Module fd4_domain_mod


Uses:
    module fd4_block_mod
    module fd4_globaldef_mod
    module fd4_vartab_mod
    module fd4_ghostlist_mod
    module fd4_mpi_mod
    module rbtree_fd4_block_mod
    module rbtree_int8_mod
    module stack_mod
    module stack3_mod
    module section_mod
Types:
    public type fd4_domain_balance
    public type fd4_domain_coord
    public type fd4_domain_blockpool
    public type fd4_domain
Variables:
    integer, public, parameter :: c2i_shft3 = 20
    integer, public, parameter :: c2i_shft2 = 10
    integer, public, parameter :: FD4_LB_HISTORY = 4
Interfaces:
    public interface fd4_domain_create
Subroutines and functions:
    public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc)
    public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc)
    public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc)
    public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc)
    public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
    private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
    private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
    public subroutine fd4_domain_clear (domain)
    public subroutine fd4_domain_delete (domain)

Module for a 3D computational domain.

Naming conventions for spatial 3D position, size, and bounds of a block, domain, etc.:

Following conversions are valid: A character prepended to a variable name for pos, ext, or bnd means: Preprocessor options:

Author: Matthias Lieber


Description of Types

fd4_domain_balance

public type fd4_domain_balance
    real (kind=r4k) :: defaultweight = 1.0_r4k
    real (kind=r4k), pointer, dimension (:,:,:) :: blkweights
    real (kind=r4k) :: localweight
    integer (kind=i_k), pointer, dimension (:) :: hidx => NULL ()
    integer (kind=i_k), pointer, dimension (:) :: rnk => NULL ()
    type (stack3) :: allblocks
    integer (kind=i_k) :: last_sfctype = -1
    real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbtimes
    integer (kind=i_k) :: curlbtime
    logical :: globalize_lbtime
    real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbbalance
    integer (kind=i_k) :: curlbbalance
    logical :: part_changed
end type fd4_domain_balance
Components:
defaultweight default weight of new created blocks (per grid cell!)
blkweights weights of all blocks
localweight sum of local block weights, updated in fd4_part_sfc
hidx hilbert sfc index of global allocated blocks
rnk ranking of global allocated blocks by hilbert sfc index
allblocks coordinates of all required blocks in global domain
last_sfctype last used type of SFC curve
lbtimes timings of last FD4_LB_HISTORY calls of load balancing
curlbtime index of current entry in lbtimes
globalize_lbtime indicate whether current lbtime needs to be globalized (max)
lbbalance balance achieved by last FD4_LB_HISTORY calls of load balancing
curlbbalance index of current entry in lbbalance
part_changed indicate whether fd4_balance_readjust tried to balance the load last time
Type to store data related to dynamic load balancing

TODO: put other domain stuff here, e.g. empties, hilbert_level


fd4_domain_coord

public type fd4_domain_coord
    integer (kind=i_k), pointer, dimension (:) :: bl => null ()
end type fd4_domain_coord
Components:
bl start indices of blocks
Type for a spatial dimension, internally used by fd4_domain

fd4_domain_blockpool

public type fd4_domain_blockpool
    type (fd4_block_link), pointer, dimension (:) :: stack => null ()
    integer (kind=i_k) :: pos = 0
end type fd4_domain_blockpool
A pool of unused blocks, implemented a fixed size stack. Internally used by fd4_domain.

fd4_domain

public type fd4_domain
    type (rbtree_fd4_block) :: blocks
    type (fd4_domain_coord), dimension (3) :: d
    integer (kind=i_k), dimension (3,2) :: dbnd
    integer (kind=i_k), dimension (3) :: blnum
    integer (kind=i_k), dimension (3,2) :: mmbsz
    integer (kind=i_k), dimension (3) :: nghosts
    logical, dimension (3) :: periodic
    type (fd4_vartab), pointer, dimension (:) :: vartab => null ()
    type (fd4_varinfo) :: varinfo
    type (fd4_ghostlist), dimension (-1:0) :: glist
    type (fd4_mpi) :: mpi
    integer, pointer, dimension (:,:,:,:) :: blkmap => null ()
    integer :: blkmap_now
    integer :: blkmap_last
    logical :: init = .false.
    integer (kind=i_k) :: hilbert_level
    integer (kind=i8k) :: partition_id
    integer (kind=i4k), pointer, dimension (:,:,:) :: empties
    integer (kind=i_k) :: blockcount = 0
    type (section) :: blockunion
    type (fd4_domain_balance) :: balance
    type (stack) :: markedblocks
    logical :: blocks_are_marked
    type (fd4_domain_blockpool), pointer, dimension (:,:,:) :: pool
    type (rbtree_int8) :: cpltypestore
    integer (kind=i4k), dimension (14) :: cplblkenc
end type fd4_domain
Components:
blocks data structure containing all blocks of the domain
d information about each of the 3 spatial dimensions
dbnd start and end indices of the grid for x,y,z coordinates
blnum number of blocks in x,y,z direction
mmbsz min and max block size in each spatial dimension
nghosts number of ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
vartab variable table
varinfo global info about variable table
glist lists of all ghost blocks (boundary + communication ghosts)
mpi MPI context
blkmap Who owns which block? Contains MPI ranks. With boundary.
blkmap_now current block map (4th dimension of blkmap)
blkmap_last last block map (4th dimension of blkmap)
init true if domain is initialized
hilbert_level level of the Hilbert SFC (depends on blnum)
partition_id unique identifier of the current partition (counter)
empties block%sempty of all blocks
blockcount global number of blocks
blockunion spatial union of bounds of all local blocks
balance Stores data related to dynamic load balancing
markedblocks list of blocks (idx) marked as to be created (global)
blocks_are_marked are there any blocks marked (global)
pool block pools for each possible block size
cpltypestore a binary tree to store MPI types for coupling
cplblkenc encoding params of index for this tree
Type for a 3D computational domain.

See also: module fd4_vartab_mod, module rbtree_fd4_block_mod


Description of Interfaces

fd4_domain_create

public interface fd4_domain_create
    module procedure fd4_domain_create_fixed
    module procedure fd4_domain_create_specific
end interface fd4_domain_create
Create new domain.

Description of Subroutines and Functions

fd4_domain_add_block

public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (3) :: bpos
    logical, intent(in) :: init
    integer (kind=i_k), intent(out) :: err
    type (fd4_block), optional, pointer :: bl
    integer (kind=i_k), optional, intent(inout) :: opt_alloc
    integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_add_block
Parameters:
domain the domain
bpos position of the block in the block decomposition
init if true, data fields are initialized with their vnull
err error status: 0...ok, -1...block already present, >0...error
bl optionally return a pointer to the new block
opt_alloc add here the number of real block allocations (no block in pool found)
opt_dealloc add here the number of real block deallocations
Add block at given block coordinates.

Allocates new block and block's static arrays. Adds the block to the block data structure. Set up the neighbor pointers of this and neighbor blocks. The value for bpos(i) must be >=1 and <=domain%blnum(i).


fd4_domain_add_ghost

public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc)
    type (fd4_domain), intent(inout) :: domain
    type (fd4_block), pointer :: b
    integer (kind=i_k), intent(in) :: dir
    integer (kind=i_k), intent(in) :: dim
    integer (kind=i_k), intent(in) :: gtype
    integer (kind=i_k), optional, intent(inout) :: opt_alloc
end subroutine fd4_domain_add_ghost
Parameters:
domain the domain
b the block
dir boundary ghost block direction
dim boundary ghost block dimension
gtype the type of ghost (FD4_GHOST_BND or FD4_GHOST_CMM)
opt_alloc add here the number of real block allocations (no block in pool found)
Internal Routine to add a boundary or comm. ghost block near b.

Note: Data fields of ghost blocks are not initialized with vnull, since is is expected that these data is read only after ghost communication.

Called from fd4_domain_add_block and in fd4_balance_readjust. The new gost block bgb is added to the ghostlist glist and a pointer from bgb is set to b:

  bgb%neigh(2,1)%l => b

Here is a list of all block components which are abused by ghost blocks:

 
  bgb%neigh(1,1)%l  ...  previous in ghostlist
  bgb%neigh(1,2)%l  ...  next in ghostlist
  bgb%neigh(2,1)%l  ...  pointer to neighor within domain (ordinary block), "parent"
  bgb%pos(1)        ...  boundary ghost block indicator, (FD4_GHOST_BND or FD4_GHOST_CMM)
  bgb%pos(2)        ...  boundary direction (1 or 2)
  bgb%pos(3)        ...  boundary dimension (1, 2 or 3)
   

TODO:


fd4_domain_delete_block

public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in) :: hidx
    integer (kind=i_k), intent(out) :: err
    integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_delete_block
Parameters:
domain the domain
hidx the index of the block on the SFC
err error status: 0...ok
opt_dealloc add here the number of real block deallocations (no space left in pool)
Delete block at given index.

Deletes all memory associated with the block, including both types of ghost blocks. Clears neighbor pointers in neighbor blocks.


fd4_domain_delete_ghost

public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc)
    type (fd4_domain), intent(inout) :: domain
    type (fd4_block), pointer :: gb
    integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_delete_ghost
Parameters:
domain the domain
gb the ghost block
opt_dealloc add here the number of real block deallocations (no space left in pool)
Internal routine to remove a boundary or comm. ghost block gb.

fd4_domain_max_bext

public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
    type (fd4_domain), intent(in) :: domain
    integer (kind=i_k), intent(out), dimension (3) :: bext
    logical, optional, intent(in) :: add_ghosts
end subroutine fd4_domain_max_bext
Parameters:
domain the domain
bext block extents greater than or equal to any block's extents, 0 on error
add_ghosts if present and true, adds ghost cells to the extent
Get the extent of an array which is large enough to hold any single block in this domain.

This is not necessarily the extent of the largest block. The allocation status of the blocks does not influence the result of this routine.


fd4_domain_create_fixed

private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (3) :: blnum
    integer (kind=i_k), intent(in), dimension (3,2) :: dbnd
    type (fd4_vartab), intent(in), dimension (:) :: vars
    integer (kind=i_k), intent(in), dimension (3) :: nghosts
    logical, intent(in), dimension (3) :: periodic
    integer (kind=i_k), intent(in) :: mpi_comm
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_fixed
Parameters:
domain the domain
blnum number of blocks in x,y,z direction
dbnd start and end indices of the grid for x,y,z coordinates
vars variable table
nghosts number of required ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
mpi_comm MPI communicator
err error status: 0...ok
Create domain with fixed block sizes.

This function calculates the actual block bounds from the number of blocks in each spatial direction and the start and end indices of the grid.

For example, the values

  dbnd(:,1) = (/ 1,-3, 1/)    ! start indices of grid in x,y,z direction
  dbnd(:,2) = (/ 9, 5, 3/)    ! end   indices of grid in x,y,z direction
  blnum     = (/ 3, 3, 3/)    ! number of blocks in x,y,z direction
result in the following block definitions used for fd4_domain_create_specific:
  blx = (/ 1,  4,  7, 10/) 
  bly = (/-3,  0,  3,  6/)
  blz = (/ 1,  2,  3,  4/)

See also: module fd4_vartab_mod


fd4_domain_create_specific

private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (:) :: blx
    integer (kind=i_k), intent(in), dimension (:) :: bly
    integer (kind=i_k), intent(in), dimension (:) :: blz
    type (fd4_vartab), intent(in), dimension (:) :: vars
    integer (kind=i_k), intent(in), dimension (3) :: nghosts
    logical, intent(in), dimension (3) :: periodic
    integer (kind=i_k), intent(in) :: mpi_comm
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_specific
Parameters:
domain the domain
blx start indices of blocks in x direction
bly start indices of blocks in y direction
blz start indices of blocks in z direction
vars variable table
nghosts number of required ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
mpi_comm MPI communicator
err error status: 0...ok
Create domain with specific block sizes.

The array blx is of length num_x_blocks + 1. The first num_x_blocks values define the start indices of the blocks in x direction. The last value defines the start index of an additional imaginary block and thus the size of the last real block. The same applies for bly and blz. The bounds of the grid is derived from the block definitions. For example

  blx = (/ 1,  4,  7, 10/) 
  bly = (/-3,  0,  3,  6/)
  blz = (/ 1,  2,  3,  4/)
define a grid with x dimension from 1 to 9, y dimension from -3 to 5 and z dimension from 1 to 3. The grid is subdivided in 3 x 3 x 3 blocks which have the size 3 x 3 x 1 grid cells.

See also: module fd4_vartab_mod


fd4_domain_clear

public subroutine fd4_domain_clear (domain)
    type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_clear
Parameters:
domain the domain
Delete all blocks of a domain.

fd4_domain_delete

public subroutine fd4_domain_delete (domain)
    type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_delete
Parameters:
domain the domain
Delete a domain.

Deallocates all data associated with domain.