Module fd4_domain_mod


Uses:
    module fd4_block_mod
    module fd4_globaldef_mod
    module fd4_vartab_mod
    module fd4_ghostlist_mod
    module fd4_mpi_mod
    module fd4_stat_mod
    module fd4_metadata_mod
    module rbtree_fd4_block_mod
    module rbtree_int8_mod
    module stack_mod
    module stack3_mod
    module section_mod
    module timing_mod
Types:
    public type fd4_domain_balance
    public type fd4_domain_lb_params
    public type fd4_domain_coord
    public type fd4_domain_blpool
    public type fd4_domain
Variables:
    integer, public, parameter :: c2i_shft3 = 20
    integer, public, parameter :: c2i_shft2 = 10
    integer, public, parameter :: FD4_LB_HISTORY = 4
Interfaces:
    public interface fd4_domain_create
Subroutines and functions:
    public subroutine fd4_domain_add_block (domain, bpos, init, alloc, dealloc, err, bl)
    public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, alloc)
    public subroutine fd4_domain_delete_block (domain, hidx, dealloc, err)
    public subroutine fd4_domain_delete_ghost (domain, gb, dealloc)
    public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
    private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
    private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
    public subroutine fd4_domain_init_metadata (domain, maxext, err)
    public subroutine fd4_domain_adapt_metadata (domain)
    public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out)
    public subroutine fd4_domain_clear (domain)
    public subroutine fd4_domain_delete (domain, opt_stats)

Module for a 3D computational domain.

Naming conventions for spatial 3D position, size, and bounds of a block, domain, etc.:

Following conversions are valid: A character prepended to a variable name for pos, ext, or bnd means: Preprocessor options:

Author: Matthias Lieber


Description of Types

fd4_domain_balance

public type fd4_domain_balance
    integer (kind=i_k), pointer, dimension (:) :: partition => NULL ()
    integer (kind=i_k), pointer, dimension (:) :: partition_old => NULL ()
    logical :: blkmap_needs_update = .false.
    real (kind=r4k) :: defaultweight = 1.0_r4k
    real (kind=r4k), pointer, dimension (:,:,:) :: blkweights => NULL ()
    real (kind=r8k), pointer, dimension (:) :: blkweights_ps => NULL ()
    real (kind=r4k) :: localweight
    integer (kind=i_k), pointer, dimension (:) :: hidx => NULL ()
    integer (kind=i_k), pointer, dimension (:) :: rnk => NULL ()
    type (stack3) :: allblocks
    integer (kind=i_k) :: last_sfctype = -1
    integer (kind=i_k) :: sfc_pstart = -1
    integer (kind=i_k), dimension (2) :: map2d
    real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbtimes
    integer (kind=i_k) :: curlbtime
    logical :: globalize_lbtime
    real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbbalance
    integer (kind=i_k) :: curlbbalance
    logical :: part_changed
    integer (kind=i_k) :: stat_check_us = -1
    integer (kind=i_k) :: stat_part_us = -1
    integer (kind=i_k) :: stat_p1d_us = -1
    integer (kind=i_k) :: stat_exwgt_us = -1
    integer (kind=i_k) :: stat_mig_us = -1
    integer (kind=i_k) :: stat_mdsd_us = -1
    integer (kind=i_k) :: stat_total_us = -1
    integer (kind=i_k) :: stat_mpi_us = -1
    integer (kind=i_k) :: stat_migblock = -1
    integer (kind=i_k) :: stat_migbytes = -1
    integer (kind=i_k) :: stat_binsteps = -1
    integer (kind=i_k) :: stat_blkmap = -1
    integer (kind=i_k) :: stat_pmrgncy = -1
end type fd4_domain_balance
Components:
partition partition vector (start indices of partitions)
partition_old old partition vector (start indices of partitions)
blkmap_needs_update true if partition is newer than domain%blkmap
defaultweight default weight of new created blocks (per grid cell!)
blkweights weights of all blocks
blkweights_ps prefix sum of all block weights for fd4_part_sfc
localweight sum of local block weights, updated in fd4_part_sfc
hidx sfc index of global allocated blocks
rnk ranking of global allocated blocks by sfc index
allblocks coordinates of all required blocks in global domain
last_sfctype last used type of SFC
sfc_pstart start index of local partition on SFC
map2d mapping of 3D to 2D for 2D SFC
lbtimes timings of last FD4_LB_HISTORY calls of load balancing
curlbtime index of current entry in lbtimes
globalize_lbtime indicate whether current lbtime needs to be globalized (max)
lbbalance balance achieved by last FD4_LB_HISTORY calls of load balancing
curlbbalance index of current entry in lbbalance
part_changed indicate whether fd4_balance_readjust tried to balance the load last time
stat_check_us index of balance checking time in the statistics object
stat_part_us index of partition calculation time in the statistics object
stat_p1d_us index of part1d (for SFC) or parmetis time in the statistics object
stat_exwgt_us index of weight exchange time in the statistics object
stat_mig_us index of migration time in the statistics object
stat_mdsd_us index of mdsd adaption time in the statistics object
stat_total_us index of total load balancing time in the statistics object
stat_mpi_us index of total MPI time of load balancing in the statistics object
stat_migblock index of migrated blocks in the statistics object
stat_migbytes index of migrated bytes in the statistics object
stat_binsteps index of maximum binary search steps for SFC methods
stat_blkmap index of number of full blkmap updates when blkmap_update=.false.
stat_pmrgncy index of number emergency block migrations in parmetis loadbalancing
Type to store data related to dynamic load balancing

TODO: put other domain stuff here, e.g. empties, sfclevel


fd4_domain_lb_params

public type fd4_domain_lb_params
    logical :: intialized = .false.
    integer (kind=i_k) :: method
    real (kind=r4k), dimension (1) :: ubvec
    real (kind=r4k) :: itr
    integer (kind=i_k) :: subset
    real (kind=r4k) :: wfactor
    logical :: use_adjwgt
    logical :: use_vsize
    integer (kind=i_k) :: sfctype
    logical :: force
    real (kind=r4k) :: lbtol
    logical :: ech
    logical :: auto
    logical :: weight
    integer (kind=i_k) :: binsteps
    real (kind=r8k) :: accuracy
    logical :: blkmap_update
end type fd4_domain_lb_params
Components:
intialized tells fd4_balance wheater the parameter have been initialized
method partitioning method (SFC, ParMetis, RCB, etc.)
ubvec ParMetis ubvec parameter
itr ParMetis itr parameter
subset ParMetis: use only subset processes for ParMetis calculation (<2...off)
wfactor ParMetis: multiply this with block weights before converting to int
use_adjwgt ParMetis: use weights for edges?
use_vsize ParMetis: use (memory size) weights for vertices?
sfctype SFC: space-filling curve type
force force to run ghost block allocation and deallocation
lbtol load balance tolerance, 0.0 (max. relaxed) ... 1.0 (strictest)
ech if false, assume that empty info did not change in any block
auto automatic decision whether load balancing is beneficial or not
weight if false ignore block weights (i.e. assume each block has same weight)
binsteps SFC: max. number of steps for binary search methods, default: 100
accuracy SFC: target ratio from best possible load balance for binary steps estimation, must be <= 1.0
blkmap_update SFC: update complete block map after partitioning, default: true
Type to store default partitioning method parameters (can be overriden when calling fd4_balance_readjust)

fd4_domain_coord

public type fd4_domain_coord
    integer (kind=i_k), pointer, dimension (:) :: bl => null ()
end type fd4_domain_coord
Components:
bl start indices of blocks
Type for a spatial dimension, internally used by fd4_domain

fd4_domain_blpool

public type fd4_domain_blpool
    type (fd4_block), pointer :: start => null ()
    integer (kind=i_k) :: len = 0
    integer (kind=i_k) :: maxlen = 0
end type fd4_domain_blpool
A pool of unused blocks, implemented as a linked list. Internally used by fd4_domain.

fd4_domain

public type fd4_domain
    type (rbtree_fd4_block) :: blocks
    type (fd4_domain_coord), dimension (3) :: d
    integer (kind=i_k), dimension (3,2) :: dbnd
    integer (kind=i_k), dimension (3) :: blnum
    integer (kind=i_k), dimension (3,2) :: mmbsz
    integer (kind=i_k), dimension (3) :: nghosts
    logical, dimension (3) :: periodic
    type (fd4_vartab), pointer, dimension (:) :: vartab => null ()
    type (fd4_varinfo) :: varinfo
    type (fd4_ghostlist), dimension (-1:0) :: glist
    type (fd4_mpi) :: mpi
    integer, pointer, dimension (:,:,:,:) :: blkmap => null ()
    integer :: blkmap_now
    integer :: blkmap_last
    logical :: init = .false.
    integer (kind=i_k) :: sfclevel
    integer (kind=i8k) :: partition_id
    integer (kind=i8k) :: partition_called
    integer (kind=i4k), pointer, dimension (:,:,:) :: empties
    integer (kind=i_k) :: blockcount = 0
    type (section) :: blockunion
    type (fd4_domain_balance) :: balance
    type (stack) :: markedblocks
    logical :: blocks_are_marked
    type (fd4_domain_blpool), pointer, dimension (:,:,:) :: pool
    type (fd4_domain_blpool), pointer, dimension (:,:,:) :: gpool
    integer (kind=i_k), dimension (3) :: gpmap
    type (rbtree_int8) :: cpltypestore
    integer (kind=i4k), dimension (14) :: cplblkenc
    type (fd4_domain_lb_params) :: lbparam
    type (fd4_stat) :: stat
    logical :: stat_enabled = .true.
    integer (kind=i_k) :: stat_bl_alloc
    integer (kind=i_k) :: stat_bl_free
    integer (kind=i_k) :: stat_gb_alloc
    integer (kind=i_k) :: stat_gb_free
    integer (kind=i_k) :: timer = TIMING_MOD_GETTIMEOFDAY
    type (fd4_metadata) :: md
    integer (kind=i_k), dimension (3,2) :: cplbnd
end type fd4_domain
Components:
blocks data structure containing all blocks of the domain
d information about each of the 3 spatial dimensions
dbnd start and end indices of the grid for x,y,z coordinates
blnum number of blocks in x,y,z direction
mmbsz min and max block size in each spatial dimension
nghosts number of ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
vartab variable table
varinfo global info about variable table
glist lists of all ghost blocks (boundary + communication ghosts)
mpi MPI context
blkmap Who owns which block? Contains MPI ranks. With boundary.
blkmap_now current block map (4th dimension of blkmap)
blkmap_last last block map (4th dimension of blkmap)
init true if domain is initialized
sfclevel level of the SFC (depends on blnum)
partition_id unique identifier of the current partition (counter)
partition_called counts up if a partitioning routine has been called
empties block%sempty of all blocks
blockcount global number of blocks
blockunion spatial union of bounds of all local blocks
balance Stores data related to dynamic load balancing
markedblocks list of blocks (idx) marked as to be created (global)
blocks_are_marked are there any blocks marked (global)
pool block pools for each possible block size
gpool ghost block pools for each possible ghost block size
gpmap mapping of ghost block size to position in gpool
cpltypestore a binary tree to store MPI types for coupling
cplblkenc encoding params of index for this tree
lbparam default load balancing method parameters
stat FD4 statistics and profiling
stat_enabled switch statistics on/off
stat_bl_alloc index of allocated blocks in the statistics object
stat_bl_free index of deallocated blocks in the statistics object
stat_gb_alloc index of allocated ghost blocks in the statistics object
stat_gb_free index of deallocated ghost blocks in the statistics object
timer timer used for fd4_iter_start_clock
md meta data for coupling
cplbnd block-bounds of local couple arrays of all couple contexts
Type for a 3D computational domain.

See also: module fd4_vartab_mod, module rbtree_fd4_block_mod


Description of Interfaces

fd4_domain_create

public interface fd4_domain_create
    module procedure fd4_domain_create_fixed
    module procedure fd4_domain_create_specific
end interface fd4_domain_create
Create new domain.

Description of Subroutines and Functions

fd4_domain_add_block

public subroutine fd4_domain_add_block (domain, bpos, init, alloc, dealloc, err, bl)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (3) :: bpos
    logical, intent(in) :: init
    integer (kind=i_k), intent(inout) :: alloc
    integer (kind=i_k), intent(inout) :: dealloc
    integer (kind=i_k), intent(out) :: err
    type (fd4_block), optional, pointer :: bl
end subroutine fd4_domain_add_block
Parameters:
domain the domain
bpos position of the block in the block decomposition
init if true, data fields are initialized with their vnull
alloc add here the number of real block allocations (no block in pool found)
dealloc add here the number of real block deallocations
err error status: 0...ok, -1...block already present, >0...error
bl optionally return a pointer to the new block
Add block at given block coordinates.

Allocates new block and block's static arrays. Adds the block to the block data structure. Set up the neighbor pointers of this and neighbor blocks. The value for bpos(i) must be >=1 and <=domain%blnum(i).


fd4_domain_add_ghost

public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, alloc)
    type (fd4_domain), intent(inout) :: domain
    type (fd4_block), pointer :: b
    integer (kind=i_k), intent(in) :: dir
    integer (kind=i_k), intent(in) :: dim
    integer (kind=i_k), intent(in) :: gtype
    integer (kind=i_k), intent(inout) :: alloc
end subroutine fd4_domain_add_ghost
Parameters:
domain the domain
b the block
dir boundary ghost block direction
dim boundary ghost block dimension
gtype the type of ghost (FD4_GHOST_BND or FD4_GHOST_CMM)
alloc add here the number of real block allocations (no block in pool found)
Internal Routine to add a boundary or comm. ghost block near b.

Note: Data fields of ghost blocks are not initialized with vnull, since is is expected that these data is read only after ghost communication.

Called from fd4_domain_add_block and in fd4_balance_readjust. The new gost block bgb is added to the ghostlist glist and a pointer from bgb is set to b:

  bgb%neigh(2,1)%l => b

Here is a list of all block components which are abused by ghost blocks:

 
  bgb%neigh(1,1)%l  ...  previous in ghostlist
  bgb%neigh(1,2)%l  ...  next in ghostlist
  bgb%neigh(2,1)%l  ...  pointer to neighor within domain (ordinary block), "parent"
  bgb%pos(1)        ...  boundary ghost block indicator, (FD4_GHOST_BND or FD4_GHOST_CMM)
  bgb%pos(2)        ...  boundary direction (1 or 2)
  bgb%pos(3)        ...  boundary dimension (1, 2 or 3)
  bgb%sempty        ...  rank that owns the remote block
   

fd4_domain_delete_block

public subroutine fd4_domain_delete_block (domain, hidx, dealloc, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in) :: hidx
    integer (kind=i_k), intent(inout) :: dealloc
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_delete_block
Parameters:
domain the domain
hidx the index of the block on the SFC
dealloc add here the number of real block deallocations (no space left in pool)
err error status: 0...ok
Delete block at given index.

Deletes all memory associated with the block, including both types of ghost blocks. Clears neighbor pointers in neighbor blocks.


fd4_domain_delete_ghost

public subroutine fd4_domain_delete_ghost (domain, gb, dealloc)
    type (fd4_domain), intent(inout) :: domain
    type (fd4_block), pointer :: gb
    integer (kind=i_k), intent(inout) :: dealloc
end subroutine fd4_domain_delete_ghost
Parameters:
domain the domain
gb the ghost block
dealloc add here the number of real block deallocations (no space left in pool)
Internal routine to remove a boundary or comm. ghost block gb.

fd4_domain_max_bext

public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
    type (fd4_domain), intent(in) :: domain
    integer (kind=i_k), intent(out), dimension (3) :: bext
    logical, optional, intent(in) :: add_ghosts
end subroutine fd4_domain_max_bext
Parameters:
domain the domain
bext block extents greater than or equal to any block's extents, 0 on error
add_ghosts if present and true, adds ghost cells to the extent
Get the extent of an array which is large enough to hold any single block in this domain.

This is not necessarily the extent of the largest block. The allocation status of the blocks does not influence the result of this routine.


fd4_domain_create_fixed

private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (3) :: blnum
    integer (kind=i_k), intent(in), dimension (3,2) :: dbnd
    type (fd4_vartab), intent(in), dimension (:) :: vars
    integer (kind=i_k), intent(in), dimension (3) :: nghosts
    logical, intent(in), dimension (3) :: periodic
    integer (kind=i_k), intent(in) :: mpi_comm
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_fixed
Parameters:
domain the domain
blnum number of blocks in x,y,z direction
dbnd start and end indices of the grid for x,y,z coordinates
vars variable table
nghosts number of required ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
mpi_comm MPI communicator
err error status: 0...ok
Create domain with fixed block sizes.

This function calculates the actual block bounds from the number of blocks in each spatial direction and the start and end indices of the grid.

For example, the values

  dbnd(:,1) = (/ 1,-3, 1/)    ! start indices of grid in x,y,z direction
  dbnd(:,2) = (/ 9, 5, 3/)    ! end   indices of grid in x,y,z direction
  blnum     = (/ 3, 3, 3/)    ! number of blocks in x,y,z direction
result in the following block definitions used for fd4_domain_create_specific:
  blx = (/ 1,  4,  7, 10/) 
  bly = (/-3,  0,  3,  6/)
  blz = (/ 1,  2,  3,  4/)

See also: module fd4_vartab_mod


fd4_domain_create_specific

private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (:) :: blx
    integer (kind=i_k), intent(in), dimension (:) :: bly
    integer (kind=i_k), intent(in), dimension (:) :: blz
    type (fd4_vartab), intent(in), dimension (:) :: vars
    integer (kind=i_k), intent(in), dimension (3) :: nghosts
    logical, intent(in), dimension (3) :: periodic
    integer (kind=i_k), intent(in) :: mpi_comm
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_specific
Parameters:
domain the domain
blx start indices of blocks in x direction
bly start indices of blocks in y direction
blz start indices of blocks in z direction
vars variable table
nghosts number of required ghost cells for each spatial dimension
periodic apply periodic boundary conditions?
mpi_comm MPI communicator
err error status: 0...ok
Create domain with specific block sizes.

The array blx is of length num_x_blocks + 1. The first num_x_blocks values define the start indices of the blocks in x direction. The last value defines the start index of an additional imaginary block and thus the size of the last real block. The same applies for bly and blz. The bounds of the grid is derived from the block definitions. For example

  blx = (/ 1,  4,  7, 10/) 
  bly = (/-3,  0,  3,  6/)
  blz = (/ 1,  2,  3,  4/)
define a grid with x dimension from 1 to 9, y dimension from -3 to 5 and z dimension from 1 to 3. The grid is subdivided in 3 x 3 x 3 blocks which have the size 3 x 3 x 1 grid cells.

See also: module fd4_vartab_mod


fd4_domain_init_metadata

public subroutine fd4_domain_init_metadata (domain, maxext, err)
    type (fd4_domain), intent(inout) :: domain
    integer (kind=i_k), intent(in), dimension (3) :: maxext
    integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_init_metadata
Parameters:
domain the domain
maxext max extent of couple arrays
err error status: 0...ok
Initialize the decomposed management of meta data for coupling.

This subroutine must be called before creating couple contexts for domain. If it is not called, no decomposed management is applied. Decomposed management is not implemented for adaptive block mode.

With maxext you can set the maximum extent of couple arrays in number of grid cells per dimension (without extension by face variables). If any value in maxext is <=0, decomposition is disabled.


fd4_domain_adapt_metadata

public subroutine fd4_domain_adapt_metadata (domain)
    type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_adapt_metadata
Parameters:
domain the domain
Optimize metadata for coupling for current local blocks.

Once this subroutine has been called, the meta data will be optimized at each repartitioning. You must first initialize decomposed management of meta date with fd4_domain_init_metadata.


fd4_domain_dump_stats

public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out)
    type (fd4_domain), intent(inout) :: domain
    integer, optional, intent(in) :: opt_root
    integer, optional, intent(in) :: opt_out
end subroutine fd4_domain_dump_stats
Parameters:
domain the domain
opt_root root MPI process that prints statistics
opt_out unit for output
Collect global min/max/avg/sum values of the FD4 statistics and let rank opt_root print the statistics to unit out. The default unit is stdout. The default root is rank 0.

fd4_domain_clear

public subroutine fd4_domain_clear (domain)
    type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_clear
Parameters:
domain the domain
Delete all blocks of a domain.

fd4_domain_delete

public subroutine fd4_domain_delete (domain, opt_stats)
    type (fd4_domain), intent(inout) :: domain
    logical, optional, intent(in) :: opt_stats
end subroutine fd4_domain_delete
Parameters:
domain the domain
opt_stats dump FD4 statistics to stdout? default is yes.
Delete a domain.

Calls fd4_domain_dump_stats and deallocates all data associated with domain.