Module fd4_couple_mod


Uses:
    module fd4_globaldef_mod
    module fd4_vartab_mod
    module fd4_block_mod
    module fd4_domain_mod
    module fd4_iter_mod
    module fd4_mpi_mod
    module fd4_mpitype_mod
    module fd4_comm_mod
    module stack_mod
    module rbtree_int_mod
    module rbtree_int8_mod
    module section_mod
Types:
    public type fd4_couple_desc
    public type fd4_couple
Variables:
    integer (kind=i_k), private, parameter :: CPL_NOFACE = 1
    integer (kind=i_k), private, parameter :: CPL_FACE = 2
    integer (kind=i_k), private, parameter :: CPL_NO = 3
    integer (kind=i_k), private, parameter :: CPL_GET = 1
    integer (kind=i_k), private, parameter :: CPL_PUT = 2
Subroutines and functions:
    public subroutine fd4_couple_create (cpl, domain, desc, err)
    private subroutine fd4_couple_build_cplarray_type (desc, bbnd, nbins, datatype, err)
    public subroutine fd4_couple_delete (cpl, err)
    public subroutine fd4_couple_put (cpl, desc, err)
    public subroutine fd4_couple_get (cpl, desc, err, opt_novnull)
    private subroutine fd4_couple_blocktype (domain, idx, st, ext, bnd, blocktype)
    public subroutine fd4_couple_mark_blocks (cpl, desc, err)
    private subroutine fd4_couple_stack_global_merge (mpi, st, err)

Coupling of FD4 to other parallel model.

Limitations:

Preprocessor options: Performance Notes: Some notes about sorting of the components of the mpi datatypes we are building here: TODO:

Author: Matthias Lieber

See also: module fd4_domain_mod


Description of Types

fd4_couple_desc

public type fd4_couple_desc
    integer (kind=i_k) :: rank
    integer (kind=i_k) :: idx
    integer (kind=i_k) :: st
    integer (kind=i_k), dimension (3,2) :: abnd
    real (kind=r_k), pointer, dimension (:,:,:) :: array => NULL ()
    real (kind=r_k), pointer, dimension (:,:,:,:) :: array4 => NULL ()
    integer (kind=i_k), dimension (3) :: offset = (/ 1, 1, 1 /)
    integer (kind=i_k) :: offset4 = 1
end type fd4_couple_desc
Components:
rank owner of this block
idx field index in vartab
st time step index
abnd bounds of the block in global coordinates
array data array
array4 4-dimensional data array (for variables with bins)
offset spatial offset of the data in array, e.g. (/1,1,1/)
offset4 offset of the data in the 4th dimension
Description of one coupling array provided by user.


fd4_couple

public type fd4_couple
    type (fd4_domain), pointer :: domain => NULL ()
    integer (kind=i_k), dimension (3,2) :: loopbnd
    integer, pointer, dimension (:,:,:,:,:) :: arraytypes
    type (fd4_mpitype), pointer, dimension (:) :: commtype
    integer (kind=i_k) :: toff
    integer (kind=i_k) :: ncplarrays
    integer (kind=i_k), pointer, dimension (:,:,:) :: rankbnd
    integer (kind=i_k), dimension (3) :: faceoffset
    integer (kind=i_k), pointer, dimension (:) :: mycplarrays
    integer (kind=i_k), pointer, dimension (:,:) :: cpldescbnd
end type fd4_couple
Components:
domain the domain
loopbnd bounds for the loop over blocks
arraytypes pre-built mpi types for couple arrays
commtype couple array mpi types for each dest rank
toff offset of types for send/receive in commtype array
ncplarrays number of local couple arrays in couple description
rankbnd union bounds of all couple arrays for each rank
faceoffset offset to local block bounds union due to face variables
mycplarrays indexes of my couple arrays in fd4_cpl_desc array
cpldescbnd first and last index of couple arrays in fd4_cpl_desc per rank
The FD4 couple context.

Description of Subroutines and Functions

fd4_couple_create

public subroutine fd4_couple_create (cpl, domain, desc, err)
    type (fd4_couple), intent(inout) :: cpl
    type (fd4_domain), intent(inout), target :: domain
    type (fd4_couple_desc), intent(in), target, dimension (:) :: desc
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_create
Parameters:
cpl couple context
domain the domain
desc description of sender arrays
err error status: 0...ok
Create FD4 couple context.

TODO:


fd4_couple_build_cplarray_type

private subroutine fd4_couple_build_cplarray_type (desc, bbnd, nbins, datatype, err)
    type (fd4_couple_desc), intent(in), target :: desc
    integer (kind=i_k), intent(in), dimension (3,2) :: bbnd
    integer (kind=i_k), intent(in) :: nbins
    integer, intent(out) :: datatype
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_build_cplarray_type
Parameters:
desc description of couple array
bbnd bounds of this (locally non-existing) block
nbins number of bins of the variable in desc
datatype the MPI data type
err error status: 0...ok
Build an MPI data type for a couple array. Internal routine.

fd4_couple_delete

public subroutine fd4_couple_delete (cpl, err)
    type (fd4_couple), intent(inout) :: cpl
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_delete
Parameters:
cpl couple context
err error status: 0...ok
Delete the couple context and all its memory.

fd4_couple_put

public subroutine fd4_couple_put (cpl, desc, err)
    type (fd4_couple), intent(inout), target :: cpl
    type (fd4_couple_desc), intent(inout), target, dimension (:) :: desc
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_put
Parameters:
cpl couple context
desc description of sender arrays
err error status: 0...ok
Put data to the framework. Each process contributes several blocks of data. desc%abnd has to be set in all processes correctly. You must supply the same desc as in fd4_couple_create, the only allowed change is the pointer to the data (desc%array).

TODO:


fd4_couple_get

public subroutine fd4_couple_get (cpl, desc, err, opt_novnull)
    type (fd4_couple), intent(inout), target :: cpl
    type (fd4_couple_desc), intent(inout), target, dimension (:) :: desc
    integer (kind=i_k), intent(out) :: err
    logical, optional, intent(in) :: opt_novnull
end subroutine fd4_couple_get
Parameters:
cpl couple context
desc description of receiver arrays
err error status: 0...ok
opt_novnull do not initalize the receiver arrays with vnull
Get data from the framework. Each process asks for several blocks of data. desc%abnd has to be set in all processes correctly.

Additional limitation for face variables: The upper face cell plane of the blocks is just cut and not transferred to the couple arrays!

TODO:


fd4_couple_blocktype

private subroutine fd4_couple_blocktype (domain, idx, st, ext, bnd, blocktype)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in) :: idx
    integer (kind=i_k), intent(in) :: st
    integer (kind=i_k), intent(in), dimension (3) :: ext
    integer (kind=i_k), intent(in), dimension (3,2) :: bnd
    integer, intent(out) :: blocktype
end subroutine fd4_couple_blocktype
Parameters:
domain the domain
idx variable index
st time step index
ext block extents
bnd bound of spatial subset
blocktype MPI data type for the block
Get an MPI data type for the variable described by (idx, st), a block of size ext, and a spatial subsection within the block described by bbnd. If an MPI data type with same properties has already been created, this type will be re-used.

Internal routine. Used in fd4_couple_put and fd4_couple_get.


fd4_couple_mark_blocks

public subroutine fd4_couple_mark_blocks (cpl, desc, err)
    type (fd4_couple), intent(inout), target :: cpl
    type (fd4_couple_desc), intent(in), target, dimension (:) :: desc
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_mark_blocks
Parameters:
cpl couple context
desc description of sender arrays
err error status: 0...ok
Pretend putting data to the framework and mark all framework blocks which need to be created to fully capture all threshold variables.

Use this in a 3-stage coupling process:

You must use the same parameters for fd4_couple_mark_blocks as for fd4_couple_put.

TODO:


fd4_couple_stack_global_merge

private subroutine fd4_couple_stack_global_merge (mpi, st, err)
    type (fd4_mpi), intent(inout) :: mpi
    type (stack), intent(inout) :: st
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_couple_stack_global_merge
Parameters:
mpi the domain's mpi context
st the stack
err error status: 0...ok
Merge the process-local stacks st to one global stack in every process.

Duplicates will be removed, the stack will be sorted. It is assumed that the process local stacks are already sorted.

Internal routine.