module fd4_block_mod module fd4_globaldef_mod module fd4_vartab_mod module fd4_ghostlist_mod module fd4_mpi_mod module fd4_stat_mod module rbtree_fd4_block_mod module rbtree_int8_mod module stack_mod module stack3_mod module section_mod module timing_modTypes:
public type fd4_domain_balance public type fd4_domain_lb_params public type fd4_domain_coord public type fd4_domain_blpool public type fd4_domainVariables:
integer, public, parameter :: c2i_shft3 = 20 integer, public, parameter :: c2i_shft2 = 10 integer, public, parameter :: FD4_LB_HISTORY = 4Interfaces:
public interface fd4_domain_createSubroutines and functions:
public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc) public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc) public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc) public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc) public subroutine fd4_domain_max_bext (domain, bext, add_ghosts) private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err) private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err) public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out) public subroutine fd4_domain_clear (domain) public subroutine fd4_domain_delete (domain, opt_stats)
Naming conventions for spatial 3D position, size, and bounds of a block, domain, etc.:
Author: Matthias Lieber
public type fd4_domain_balance real (kind=r4k) :: defaultweight = 1.0_r4k real (kind=r4k), pointer, dimension (:,:,:) :: blkweights real (kind=r4k) :: localweight integer (kind=i_k), pointer, dimension (:) :: hidx => NULL () integer (kind=i_k), pointer, dimension (:) :: rnk => NULL () type (stack3) :: allblocks integer (kind=i_k) :: last_sfctype = -1 real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbtimes integer (kind=i_k) :: curlbtime logical :: globalize_lbtime real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbbalance integer (kind=i_k) :: curlbbalance logical :: part_changed integer (kind=i_k) :: stat_check_us integer (kind=i_k) :: stat_part_us integer (kind=i_k) :: stat_mig_us integer (kind=i_k) :: stat_migblock integer (kind=i_k) :: stat_migbytes end type fd4_domain_balanceComponents:
defaultweight | default weight of new created blocks (per grid cell!) |
blkweights | weights of all blocks |
localweight | sum of local block weights, updated in fd4_part_sfc |
hidx | hilbert sfc index of global allocated blocks |
rnk | ranking of global allocated blocks by hilbert sfc index |
allblocks | coordinates of all required blocks in global domain |
last_sfctype | last used type of SFC curve |
lbtimes | timings of last FD4_LB_HISTORY calls of load balancing |
curlbtime | index of current entry in lbtimes |
globalize_lbtime | indicate whether current lbtime needs to be globalized (max) |
lbbalance | balance achieved by last FD4_LB_HISTORY calls of load balancing |
curlbbalance | index of current entry in lbbalance |
part_changed | indicate whether fd4_balance_readjust tried to balance the load last time |
stat_check_us | index of balance checking time in the statistics object |
stat_part_us | index of partition calculation time in the statistics object |
stat_mig_us | index of migration time in the statistics object |
stat_migblock | index of migrated blocks in the statistics object |
stat_migbytes | index of migrated bytes in the statistics object |
TODO: put other domain stuff here, e.g. empties, hilbert_level
public type fd4_domain_lb_params logical :: intialized = .false. integer (kind=i_k) :: method real (kind=r4k), dimension (1) :: ubvec real (kind=r4k) :: itr integer (kind=i_k) :: subset real (kind=r4k) :: wfactor integer (kind=i_k) :: sfctype logical :: force real (kind=r4k) :: lbtol logical :: ech logical :: auto logical :: weight end type fd4_domain_lb_paramsComponents:
intialized | tells fd4_balance wheater the parameter have been initialized |
method | partitioning method (SFC, ParMetis, RCB, etc.) |
ubvec | ParMetis ubvec parameter |
itr | ParMetis itr parameter |
subset | ParMetis: use only subset processes for ParMetis calculation (<2...off) |
wfactor | ParMetis: multiply this with block weights before converting to int |
sfctype | SFC: space-filling curve type |
force | force to run ghost block allocation and deallocation |
lbtol | load balance tolerance, 0.0 (max. relaxed) ... 1.0 (strictest) |
ech | if false, assume that empty info did not change in any block |
auto | automatic decision whether load balancing is beneficial or not |
weight | if false ignore block weights (i.e. assume each block has same weight) |
public type fd4_domain_coord integer (kind=i_k), pointer, dimension (:) :: bl => null () end type fd4_domain_coordComponents:
bl | start indices of blocks |
public type fd4_domain_blpool type (fd4_block), pointer :: start => null () integer (kind=i_k) :: len = 0 integer (kind=i_k) :: maxlen = 0 end type fd4_domain_blpoolA pool of unused blocks, implemented as a linked list. Internally used by fd4_domain.
public type fd4_domain type (rbtree_fd4_block) :: blocks type (fd4_domain_coord), dimension (3) :: d integer (kind=i_k), dimension (3,2) :: dbnd integer (kind=i_k), dimension (3) :: blnum integer (kind=i_k), dimension (3,2) :: mmbsz integer (kind=i_k), dimension (3) :: nghosts logical, dimension (3) :: periodic type (fd4_vartab), pointer, dimension (:) :: vartab => null () type (fd4_varinfo) :: varinfo type (fd4_ghostlist), dimension (-1:0) :: glist type (fd4_mpi) :: mpi integer, pointer, dimension (:,:,:,:) :: blkmap => null () integer :: blkmap_now integer :: blkmap_last logical :: init = .false. integer (kind=i_k) :: hilbert_level integer (kind=i8k) :: partition_id integer (kind=i4k), pointer, dimension (:,:,:) :: empties integer (kind=i_k) :: blockcount = 0 type (section) :: blockunion type (fd4_domain_balance) :: balance type (stack) :: markedblocks logical :: blocks_are_marked type (fd4_domain_blpool), pointer, dimension (:,:,:) :: pool type (fd4_domain_blpool), pointer, dimension (:,:,:) :: gpool integer (kind=i_k), dimension (3) :: gpmap type (rbtree_int8) :: cpltypestore integer (kind=i4k), dimension (14) :: cplblkenc type (fd4_domain_lb_params) :: lbparam type (fd4_stat) :: stat logical :: stat_enabled = .true. integer (kind=i_k) :: stat_bl_alloc integer (kind=i_k) :: stat_bl_free integer (kind=i_k) :: stat_gb_alloc integer (kind=i_k) :: stat_gb_free integer (kind=i_k) :: timer = TIMING_MOD_GETTIMEOFDAY end type fd4_domainComponents:
blocks | data structure containing all blocks of the domain |
d | information about each of the 3 spatial dimensions |
dbnd | start and end indices of the grid for x,y,z coordinates |
blnum | number of blocks in x,y,z direction |
mmbsz | min and max block size in each spatial dimension |
nghosts | number of ghost cells for each spatial dimension |
periodic | apply periodic boundary conditions? |
vartab | variable table |
varinfo | global info about variable table |
glist | lists of all ghost blocks (boundary + communication ghosts) |
mpi | MPI context |
blkmap | Who owns which block? Contains MPI ranks. With boundary. |
blkmap_now | current block map (4th dimension of blkmap) |
blkmap_last | last block map (4th dimension of blkmap) |
init | true if domain is initialized |
hilbert_level | level of the Hilbert SFC (depends on blnum) |
partition_id | unique identifier of the current partition (counter) |
empties | block%sempty of all blocks |
blockcount | global number of blocks |
blockunion | spatial union of bounds of all local blocks |
balance | Stores data related to dynamic load balancing |
markedblocks | list of blocks (idx) marked as to be created (global) |
blocks_are_marked | are there any blocks marked (global) |
pool | block pools for each possible block size |
gpool | ghost block pools for each possible ghost block size |
gpmap | mapping of ghost block size to position in gpool |
cpltypestore | a binary tree to store MPI types for coupling |
cplblkenc | encoding params of index for this tree |
lbparam | default load balancing method parameters |
stat | FD4 statistics and profiling |
stat_enabled | switch statistics on/off |
stat_bl_alloc | index of allocated blocks in the statistics object |
stat_bl_free | index of deallocated blocks in the statistics object |
stat_gb_alloc | index of allocated ghost blocks in the statistics object |
stat_gb_free | index of deallocated ghost blocks in the statistics object |
timer | timer used for fd4_iter_start_clock |
See also: module fd4_vartab_mod, module rbtree_fd4_block_mod
public interface fd4_domain_create module procedure fd4_domain_create_fixed module procedure fd4_domain_create_specific end interface fd4_domain_createCreate new domain.
public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc) type (fd4_domain), intent(inout) :: domain integer (kind=i_k), intent(in), dimension (3) :: bpos logical, intent(in) :: init integer (kind=i_k), intent(out) :: err type (fd4_block), optional, pointer :: bl integer (kind=i_k), optional, intent(inout) :: opt_alloc integer (kind=i_k), optional, intent(inout) :: opt_dealloc end subroutine fd4_domain_add_blockParameters:
domain | the domain |
bpos | position of the block in the block decomposition |
init | if true, data fields are initialized with their vnull |
err | error status: 0...ok, -1...block already present, >0...error |
bl | optionally return a pointer to the new block |
opt_alloc | add here the number of real block allocations (no block in pool found) |
opt_dealloc | add here the number of real block deallocations |
Allocates new block and block's static arrays. Adds the block to the block data structure. Set up the neighbor pointers of this and neighbor blocks. The value for bpos(i) must be >=1 and <=domain%blnum(i).
public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc) type (fd4_domain), intent(inout) :: domain type (fd4_block), pointer :: b integer (kind=i_k), intent(in) :: dir integer (kind=i_k), intent(in) :: dim integer (kind=i_k), intent(in) :: gtype integer (kind=i_k), optional, intent(inout) :: opt_alloc end subroutine fd4_domain_add_ghostParameters:
domain | the domain |
b | the block |
dir | boundary ghost block direction |
dim | boundary ghost block dimension |
gtype | the type of ghost (FD4_GHOST_BND or FD4_GHOST_CMM) |
opt_alloc | add here the number of real block allocations (no block in pool found) |
Note: Data fields of ghost blocks are not initialized with vnull, since is is expected that these data is read only after ghost communication.
Called from fd4_domain_add_block and in fd4_balance_readjust. The new gost block bgb is added to the ghostlist glist and a pointer from bgb is set to b:
bgb%neigh(2,1)%l => b
Here is a list of all block components which are abused by ghost blocks:
bgb%neigh(1,1)%l ... previous in ghostlist bgb%neigh(1,2)%l ... next in ghostlist bgb%neigh(2,1)%l ... pointer to neighor within domain (ordinary block), "parent" bgb%pos(1) ... boundary ghost block indicator, (FD4_GHOST_BND or FD4_GHOST_CMM) bgb%pos(2) ... boundary direction (1 or 2) bgb%pos(3) ... boundary dimension (1, 2 or 3)
public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc) type (fd4_domain), intent(inout) :: domain integer (kind=i_k), intent(in) :: hidx integer (kind=i_k), intent(out) :: err integer (kind=i_k), optional, intent(inout) :: opt_dealloc end subroutine fd4_domain_delete_blockParameters:
domain | the domain |
hidx | the index of the block on the SFC |
err | error status: 0...ok |
opt_dealloc | add here the number of real block deallocations (no space left in pool) |
Deletes all memory associated with the block, including both types of ghost blocks. Clears neighbor pointers in neighbor blocks.
public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc) type (fd4_domain), intent(inout) :: domain type (fd4_block), pointer :: gb integer (kind=i_k), optional, intent(inout) :: opt_dealloc end subroutine fd4_domain_delete_ghostParameters:
domain | the domain |
gb | the ghost block |
opt_dealloc | add here the number of real block deallocations (no space left in pool) |
public subroutine fd4_domain_max_bext (domain, bext, add_ghosts) type (fd4_domain), intent(in) :: domain integer (kind=i_k), intent(out), dimension (3) :: bext logical, optional, intent(in) :: add_ghosts end subroutine fd4_domain_max_bextParameters:
domain | the domain |
bext | block extents greater than or equal to any block's extents, 0 on error |
add_ghosts | if present and true, adds ghost cells to the extent |
This is not necessarily the extent of the largest block. The allocation status of the blocks does not influence the result of this routine.
private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err) type (fd4_domain), intent(inout) :: domain integer (kind=i_k), intent(in), dimension (3) :: blnum integer (kind=i_k), intent(in), dimension (3,2) :: dbnd type (fd4_vartab), intent(in), dimension (:) :: vars integer (kind=i_k), intent(in), dimension (3) :: nghosts logical, intent(in), dimension (3) :: periodic integer (kind=i_k), intent(in) :: mpi_comm integer (kind=i_k), intent(out) :: err end subroutine fd4_domain_create_fixedParameters:
domain | the domain |
blnum | number of blocks in x,y,z direction |
dbnd | start and end indices of the grid for x,y,z coordinates |
vars | variable table |
nghosts | number of required ghost cells for each spatial dimension |
periodic | apply periodic boundary conditions? |
mpi_comm | MPI communicator |
err | error status: 0...ok |
This function calculates the actual block bounds from the number of blocks in each spatial direction and the start and end indices of the grid.
For example, the values
dbnd(:,1) = (/ 1,-3, 1/) ! start indices of grid in x,y,z direction dbnd(:,2) = (/ 9, 5, 3/) ! end indices of grid in x,y,z direction blnum = (/ 3, 3, 3/) ! number of blocks in x,y,z directionresult in the following block definitions used for fd4_domain_create_specific:
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)
See also: module fd4_vartab_mod
private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err) type (fd4_domain), intent(inout) :: domain integer (kind=i_k), intent(in), dimension (:) :: blx integer (kind=i_k), intent(in), dimension (:) :: bly integer (kind=i_k), intent(in), dimension (:) :: blz type (fd4_vartab), intent(in), dimension (:) :: vars integer (kind=i_k), intent(in), dimension (3) :: nghosts logical, intent(in), dimension (3) :: periodic integer (kind=i_k), intent(in) :: mpi_comm integer (kind=i_k), intent(out) :: err end subroutine fd4_domain_create_specificParameters:
domain | the domain |
blx | start indices of blocks in x direction |
bly | start indices of blocks in y direction |
blz | start indices of blocks in z direction |
vars | variable table |
nghosts | number of required ghost cells for each spatial dimension |
periodic | apply periodic boundary conditions? |
mpi_comm | MPI communicator |
err | error status: 0...ok |
The array blx is of length num_x_blocks + 1. The first num_x_blocks values define the start indices of the blocks in x direction. The last value defines the start index of an additional imaginary block and thus the size of the last real block. The same applies for bly and blz. The bounds of the grid is derived from the block definitions. For example
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)define a grid with x dimension from 1 to 9, y dimension from -3 to 5 and z dimension from 1 to 3. The grid is subdivided in 3 x 3 x 3 blocks which have the size 3 x 3 x 1 grid cells.
See also: module fd4_vartab_mod
public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out) type (fd4_domain), intent(inout) :: domain integer, optional, intent(in) :: opt_root integer, optional, intent(in) :: opt_out end subroutine fd4_domain_dump_statsParameters:
domain | the domain |
opt_root | root MPI process that prints statistics |
opt_out | unit for output |
public subroutine fd4_domain_clear (domain) type (fd4_domain), intent(inout) :: domain end subroutine fd4_domain_clearParameters:
domain | the domain |
public subroutine fd4_domain_delete (domain, opt_stats) type (fd4_domain), intent(inout) :: domain logical, optional, intent(in) :: opt_stats end subroutine fd4_domain_deleteParameters:
domain | the domain |
opt_stats | dump FD4 statistics to stdout? default is yes. |
Calls fd4_domain_dump_stats and deallocates all data associated with domain.