module fd4_block_mod
module fd4_globaldef_mod
module fd4_vartab_mod
module fd4_ghostlist_mod
module fd4_mpi_mod
module fd4_stat_mod
module fd4_metadata_mod
module fd4_part_dir_mod
module rbtree_fd4_block_mod
module rbtree_int8_mod
module stack_mod
module stack3_mod
module section_mod
module timing_mod
module zoltan
Types:
public type fd4_domain_balance
public type fd4_domain_lb_params
public type fd4_domain_coord
public type fd4_domain_blpool
public type fd4_domain
Variables:
integer, public, parameter :: c2i_shft3 = FD4_DOMAIN_C2I_SHFT3
integer, public, parameter :: c2i_shft2 = FD4_DOMAIN_C2I_SHFT2
integer, public, parameter :: FD4_LB_HISTORY = 4
Interfaces:
public interface fd4_domain_create
Subroutines and functions:
public subroutine fd4_domain_add_block (domain, bpos, init, alloc, dealloc, err, bl)
public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, alloc)
public subroutine fd4_domain_delete_block (domain, hidx, dealloc, err)
public subroutine fd4_domain_delete_ghost (domain, gb, dealloc)
public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
public subroutine fd4_domain_init_metadata (domain, maxext, err)
public subroutine fd4_domain_adapt_metadata (domain)
public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out)
public subroutine fd4_domain_clear (domain)
public subroutine fd4_domain_delete (domain, opt_stats)
Naming conventions for spatial 3D position, size, and bounds of a block, domain, etc.:
Author: Matthias Lieber
public type fd4_domain_balance
integer (kind=i_k), pointer, dimension (:) :: partition => NULL ()
integer (kind=i_k), pointer, dimension (:) :: partition_old => NULL ()
logical :: blkmap_needs_update = .false.
real (kind=rwk) :: defaultweight = 1.0_rwk
real (kind=rwk), pointer, dimension (:,:,:) :: blkweights => NULL ()
real (kind=r8k), pointer, dimension (:) :: blkweights_ps => NULL ()
real (kind=r8k) :: localweight = 0.0_r8k
integer (kind=i_k), pointer, dimension (:) :: hidx => NULL ()
integer (kind=i_k), pointer, dimension (:) :: rnk => NULL ()
type (stack3) :: allblocks
integer (kind=i_k) :: last_sfctype = -1
integer (kind=i_k) :: sfc_pstart = -1
integer (kind=i_k), dimension (2) :: map2d
real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbtimes
integer (kind=i_k) :: curlbtime
logical :: globalize_lbtime
real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbbalance
integer (kind=i_k) :: curlbbalance
logical :: part_changed
integer (kind=i_k) :: sfcd_ngroups = 0
integer (kind=i_k) :: sfcd_groupsz = 0
integer (kind=i_k) :: sfcd_mygroup = -1
type (fd4_mpi) :: sfcd_mpi
integer :: comm_sfcd_master = MPI_COMM_NULL
real (kind=r8k), pointer, dimension (:) :: sfcd_blkweights_ps => NULL ()
type (Zoltan_Struct), pointer :: zstruct => NULL ()
integer :: zcomm = MPI_COMM_NULL
integer :: znum_imp = 0
type (fd4_part_dir) :: partdir
integer (kind=i_k) :: stat_check_us = -1
integer (kind=i_k) :: stat_part_us = -1
integer (kind=i_k) :: stat_init_us = -1
integer (kind=i_k) :: stat_exwgt_us = -1
integer (kind=i_k) :: stat_pfxs_us = -1
integer (kind=i_k) :: stat_p1dc_us = -1
integer (kind=i_k) :: stat_bcbr_us = -1
integer (kind=i_k) :: stat_p1d_us = -1
integer (kind=i_k) :: stat_dspv_us = -1
integer (kind=i_k) :: stat_blkm_us = -1
integer (kind=i_k) :: stat_mig_us = -1
integer (kind=i_k) :: stat_mdsd_us = -1
integer (kind=i_k) :: stat_total_us = -1
integer (kind=i_k) :: stat_mpi_us = -1
integer (kind=i_k) :: stat_migblock = -1
integer (kind=i_k) :: stat_migbytes = -1
integer (kind=i_k) :: stat_binsteps = -1
integer (kind=i_k) :: stat_blkmap = -1
integer (kind=i_k) :: stat_pmrgncy = -1
end type fd4_domain_balance
Components:| partition | partition vector (start indices of partitions) |
| partition_old | old partition vector (start indices of partitions) |
| blkmap_needs_update | true if partition is newer than domain%blkmap |
| defaultweight | default weight of new created blocks (per grid cell!) |
| blkweights | weights of all blocks |
| blkweights_ps | prefix sum of all block weights for 1D partitioning |
| localweight | sum of local block weights, updated in fd4_part_sfc |
| hidx | sfc index of global allocated blocks |
| rnk | ranking of global allocated blocks by sfc index |
| allblocks | coordinates of all required blocks in global domain |
| last_sfctype | last used type of SFC |
| sfc_pstart | start index of local partition on SFC |
| map2d | mapping of 3D to 2D for 2D SFC |
| lbtimes | timings of last FD4_LB_HISTORY calls of load balancing |
| curlbtime | index of current entry in lbtimes |
| globalize_lbtime | indicate whether current lbtime needs to be globalized (max) |
| lbbalance | balance achieved by last FD4_LB_HISTORY calls of load balancing |
| curlbbalance | index of current entry in lbbalance |
| part_changed | indicate whether fd4_balance_readjust tried to balance the load last time |
| sfcd_ngroups | number of SFC-D groups |
| sfcd_groupsz | SFC-D group size (sfcd_groupsz * sfcd_ngroups = nprocs) |
| sfcd_mygroup | id of this rank's SFC-D group (0...ngroups-1) |
| sfcd_mpi | MPI context for this SFC-D group (comm, rank, nprocs) |
| comm_sfcd_master | MPI communicator containing all group masters |
| sfcd_blkweights_ps | global prefix sum of local block weights for SFC-D |
| stat_check_us | balance checking time |
| stat_part_us | partition calculation time |
| stat_init_us | SFCD: initialization time (MPI_Comm_split) |
| stat_exwgt_us | weight exchange time |
| stat_pfxs_us | prefix sum (for SFC) time |
| stat_p1dc_us | SFCD: coarse-grained part1d |
| stat_bcbr_us | SFCD: comm and broadcast of coarse partition borders |
| stat_p1d_us | part1d (for SFC) or parmetis time |
| stat_dspv_us | SFCD: distribution of final partition vector |
| stat_blkm_us | blkmap update (for SFC) time |
| stat_mig_us | migration time |
| stat_mdsd_us | mdsd adaption time |
| stat_total_us | total load balancing time |
| stat_mpi_us | total MPI time of load balancing |
| stat_migblock | migrated blocks |
| stat_migbytes | migrated bytes |
| stat_binsteps | maximum binary search steps for SFC methods |
| stat_blkmap | number of full blkmap updates when blkmap_update=.false. |
| stat_pmrgncy | number emergency block migrations in parmetis loadbalancing |
TODO: put other domain stuff here, e.g. empties, sfclevel
public type fd4_domain_lb_params
logical :: intialized = .false.
integer (kind=i_k) :: method
real (kind=r4k), dimension (1) :: ubvec
real (kind=r4k) :: itr
integer (kind=i_k) :: subset
real (kind=r4k) :: wfactor
logical :: use_adjwgt
logical :: use_vsize
integer (kind=i_k) :: sfctype
logical :: force
real (kind=r4k) :: lbtol
logical :: ech
logical :: auto
logical :: weight
integer (kind=i_k) :: binsteps
real (kind=r8k) :: accuracy
logical :: blkmap_update
integer (kind=i_k) :: groupsz
integer (kind=i_k) :: wgtxmeth
integer (kind=i_k) :: dp1dmeth
integer (kind=i_k) :: dspvmeth
logical :: par_pfxs
integer (kind=i_k) :: p1d_root
logical :: mwgtcomm
logical :: spartvec
end type fd4_domain_lb_params
Components:| intialized | tells fd4_balance wheater the parameter have been initialized |
| method | partitioning method (SFC, ParMetis, RCB, etc.) |
| ubvec | ParMetis ubvec parameter |
| itr | ParMetis itr parameter |
| subset | ParMetis: use only subset processes for ParMetis calculation (<2...off) |
| wfactor | ParMetis: multiply this with block weights before converting to int |
| use_adjwgt | ParMetis: use weights for edges? |
| use_vsize | ParMetis: use (memory size) weights for vertices? |
| sfctype | SFC: space-filling curve type |
| force | force to run ghost block allocation and deallocation |
| lbtol | load balance tolerance, 0.0 (max. relaxed) ... 1.0 (strictest) |
| ech | if false, assume that empty info did not change in any block |
| auto | automatic decision whether load balancing is beneficial or not |
| weight | if false ignore block weights (i.e. assume each block has same weight) |
| binsteps | SFC: max. number of steps for binary search methods, default: 100 |
| accuracy | SFC: target ratio from best possible load balance for binary steps estimation, must be <= 1.0 |
| blkmap_update | SFC: update complete block map after partitioning, default: false |
| groupsz | SFCD: group size for distributed/hierarchical method |
| wgtxmeth | SFCD: method for exchanging block weights |
| dp1dmeth | SFCD: 1D partitioning used within the groups, default: FD4_PART_SFC_PARALLEL |
| dspvmeth | SFCD: partition vector distribution method, default: FD4_PART_SFCD_SCATTER_ALLGATHER |
| par_pfxs | SFC: compute prefix sum in parallel before gathering of weights, default: false |
| p1d_root | SFC: compute 1D-partitioning on rank p1d_root only, default: -1 (all ranks) |
| mwgtcomm | SFC: determine maxweight using an MPI reduce (binary search methods only), default: true |
| spartvec | SFC: gather partition vector sparsely only - experimental, default: false |
public type fd4_domain_coord
integer (kind=i_k), pointer, dimension (:) :: bl => null ()
end type fd4_domain_coord
Components:| bl | start indices of blocks |
public type fd4_domain_blpool
type (fd4_block), pointer :: start => null ()
integer (kind=i_k) :: len = 0
integer (kind=i_k) :: maxlen = 0
end type fd4_domain_blpool
A pool of unused blocks, implemented as a linked list. Internally used by fd4_domain.
public type fd4_domain
type (rbtree_fd4_block) :: blocks
type (fd4_domain_coord), dimension (3) :: d
integer (kind=i_k), dimension (3,2) :: dbnd
integer (kind=i_k), dimension (3) :: blnum
integer (kind=i_k), dimension (3,2) :: mmbsz
integer (kind=i_k), dimension (3) :: nghosts
logical, dimension (3) :: periodic
type (fd4_vartab), pointer, dimension (:) :: vartab => null ()
type (fd4_varinfo) :: varinfo
type (fd4_ghostlist), dimension (-1:0) :: glist
type (fd4_mpi) :: mpi
integer, pointer, dimension (:,:,:,:) :: blkmap => null ()
integer :: blkmap_now
integer :: blkmap_last
logical :: init = .false.
integer (kind=i_k) :: sfclevel
integer (kind=i8k) :: partition_id
integer (kind=i8k) :: partition_called
integer (kind=i4k), pointer, dimension (:,:,:) :: empties
integer (kind=i_k) :: blockcount = 0
type (section) :: blockunion
type (fd4_domain_balance) :: balance
type (stack) :: markedblocks
logical :: blocks_are_marked
type (fd4_domain_blpool), pointer, dimension (:,:,:) :: pool
type (fd4_domain_blpool), pointer, dimension (:,:,:) :: gpool
integer (kind=i_k), dimension (3) :: gpmap
type (rbtree_int8) :: cpltypestore
integer (kind=i4k), dimension (14) :: cplblkenc
type (fd4_domain_lb_params) :: lbparam
type (fd4_stat) :: stat
logical :: stat_enabled = .true.
integer (kind=i_k) :: stat_bl_alloc
integer (kind=i_k) :: stat_bl_free
integer (kind=i_k) :: stat_gb_alloc
integer (kind=i_k) :: stat_gb_free
integer (kind=i_k) :: timer = TIMING_MOD_GETTIMEOFDAY
type (fd4_metadata) :: md
integer (kind=i_k), dimension (3,2) :: cplbnd
end type fd4_domain
Components:| blocks | data structure containing all blocks of the domain |
| d | information about each of the 3 spatial dimensions |
| dbnd | start and end indices of the grid for x,y,z coordinates |
| blnum | number of blocks in x,y,z direction |
| mmbsz | min and max block size in each spatial dimension |
| nghosts | number of ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| vartab | variable table |
| varinfo | global info about variable table |
| glist | lists of all ghost blocks (boundary + communication ghosts) |
| mpi | MPI context |
| blkmap | Who owns which block? Contains MPI ranks. With boundary. |
| blkmap_now | current block map (4th dimension of blkmap) |
| blkmap_last | last block map (4th dimension of blkmap) |
| init | true if domain is initialized |
| sfclevel | level of the SFC (depends on blnum) |
| partition_id | unique identifier of the current partition (counter) |
| partition_called | counts up if a partitioning routine has been called |
| empties | block%sempty of all blocks |
| blockcount | global number of blocks |
| blockunion | spatial union of bounds of all local blocks |
| balance | Stores data related to dynamic load balancing |
| markedblocks | list of blocks (idx) marked as to be created (global) |
| blocks_are_marked | are there any blocks marked (global) |
| pool | block pools for each possible block size |
| gpool | ghost block pools for each possible ghost block size |
| gpmap | mapping of ghost block size to position in gpool |
| cpltypestore | a binary tree to store MPI types for coupling |
| cplblkenc | encoding params of index for this tree |
| lbparam | default load balancing method parameters |
| stat | FD4 statistics and profiling |
| stat_enabled | switch statistics on/off |
| stat_bl_alloc | index of allocated blocks in the statistics object |
| stat_bl_free | index of deallocated blocks in the statistics object |
| stat_gb_alloc | index of allocated ghost blocks in the statistics object |
| stat_gb_free | index of deallocated ghost blocks in the statistics object |
| timer | timer used for fd4_iter_start_clock |
| md | meta data for coupling |
| cplbnd | block-bounds of local couple arrays of all couple contexts |
See also: module fd4_vartab_mod, module rbtree_fd4_block_mod
public interface fd4_domain_create
module procedure fd4_domain_create_fixed
module procedure fd4_domain_create_specific
end interface fd4_domain_create
Create new domain.
public subroutine fd4_domain_add_block (domain, bpos, init, alloc, dealloc, err, bl)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (3) :: bpos
logical, intent(in) :: init
integer (kind=i_k), intent(inout) :: alloc
integer (kind=i_k), intent(inout) :: dealloc
integer (kind=i_k), intent(out) :: err
type (fd4_block), optional, pointer :: bl
end subroutine fd4_domain_add_block
Parameters:| domain | the domain |
| bpos | position of the block in the block decomposition |
| init | if true, data fields are initialized with their vnull |
| alloc | add here the number of real block allocations (no block in pool found) |
| dealloc | add here the number of real block deallocations |
| err | error status: 0...ok, -1...block already present, >0...error |
| bl | optionally return a pointer to the new block |
Allocates new block and block's static arrays. Adds the block to the block data structure. Set up the neighbor pointers of this and neighbor blocks. The value for bpos(i) must be >=1 and <=domain%blnum(i).
public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, alloc)
type (fd4_domain), intent(inout) :: domain
type (fd4_block), pointer :: b
integer (kind=i_k), intent(in) :: dir
integer (kind=i_k), intent(in) :: dim
integer (kind=i_k), intent(in) :: gtype
integer (kind=i_k), intent(inout) :: alloc
end subroutine fd4_domain_add_ghost
Parameters:| domain | the domain |
| b | the block |
| dir | boundary ghost block direction |
| dim | boundary ghost block dimension |
| gtype | the type of ghost (FD4_GHOST_BND or FD4_GHOST_CMM) |
| alloc | add here the number of real block allocations (no block in pool found) |
Note: Data fields of ghost blocks are not initialized with vnull, since is is expected that these data is read only after ghost communication.
Called from fd4_domain_add_block and in fd4_balance_readjust. The new gost block bgb is added to the ghostlist glist and a pointer from bgb is set to b:
bgb%neigh(2,1)%l => b
Here is a list of all block components which are abused by ghost blocks:
bgb%neigh(1,1)%l ... previous in ghostlist bgb%neigh(1,2)%l ... next in ghostlist bgb%neigh(2,1)%l ... pointer to neighor within domain (ordinary block), "parent" bgb%pos(1) ... boundary ghost block indicator, (FD4_GHOST_BND or FD4_GHOST_CMM) bgb%pos(2) ... boundary direction (1 or 2) bgb%pos(3) ... boundary dimension (1, 2 or 3) bgb%sempty ... rank that owns the remote block
public subroutine fd4_domain_delete_block (domain, hidx, dealloc, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in) :: hidx
integer (kind=i_k), intent(inout) :: dealloc
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_delete_block
Parameters:| domain | the domain |
| hidx | the index of the block on the SFC |
| dealloc | add here the number of real block deallocations (no space left in pool) |
| err | error status: 0...ok |
Deletes all memory associated with the block, including both types of ghost blocks. Clears neighbor pointers in neighbor blocks.
public subroutine fd4_domain_delete_ghost (domain, gb, dealloc)
type (fd4_domain), intent(inout) :: domain
type (fd4_block), pointer :: gb
integer (kind=i_k), intent(inout) :: dealloc
end subroutine fd4_domain_delete_ghost
Parameters:| domain | the domain |
| gb | the ghost block |
| dealloc | add here the number of real block deallocations (no space left in pool) |
public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
type (fd4_domain), intent(in) :: domain
integer (kind=i_k), intent(out), dimension (3) :: bext
logical, optional, intent(in) :: add_ghosts
end subroutine fd4_domain_max_bext
Parameters:| domain | the domain |
| bext | block extents greater than or equal to any block's extents, 0 on error |
| add_ghosts | if present and true, adds ghost cells to the extent |
This is not necessarily the extent of the largest block. The allocation status of the blocks does not influence the result of this routine.
private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (3) :: blnum
integer (kind=i_k), intent(in), dimension (3,2) :: dbnd
type (fd4_vartab), intent(in), dimension (:) :: vars
integer (kind=i_k), intent(in), dimension (3) :: nghosts
logical, intent(in), dimension (3) :: periodic
integer (kind=i_k), intent(in) :: mpi_comm
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_fixed
Parameters:| domain | the domain |
| blnum | number of blocks in x,y,z direction |
| dbnd | start and end indices of the grid for x,y,z coordinates |
| vars | variable table |
| nghosts | number of required ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| mpi_comm | MPI communicator |
| err | error status: 0...ok |
This function calculates the actual block bounds from the number of blocks in each spatial direction and the start and end indices of the grid.
For example, the values
dbnd(:,1) = (/ 1,-3, 1/) ! start indices of grid in x,y,z direction dbnd(:,2) = (/ 9, 5, 3/) ! end indices of grid in x,y,z direction blnum = (/ 3, 3, 3/) ! number of blocks in x,y,z directionresult in the following block definitions used for fd4_domain_create_specific:
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)
See also: module fd4_vartab_mod
private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (:) :: blx
integer (kind=i_k), intent(in), dimension (:) :: bly
integer (kind=i_k), intent(in), dimension (:) :: blz
type (fd4_vartab), intent(in), dimension (:) :: vars
integer (kind=i_k), intent(in), dimension (3) :: nghosts
logical, intent(in), dimension (3) :: periodic
integer (kind=i_k), intent(in) :: mpi_comm
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_specific
Parameters:| domain | the domain |
| blx | start indices of blocks in x direction |
| bly | start indices of blocks in y direction |
| blz | start indices of blocks in z direction |
| vars | variable table |
| nghosts | number of required ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| mpi_comm | MPI communicator |
| err | error status: 0...ok |
The array blx is of length num_x_blocks + 1. The first num_x_blocks values define the start indices of the blocks in x direction. The last value defines the start index of an additional imaginary block and thus the size of the last real block. The same applies for bly and blz. The bounds of the grid is derived from the block definitions. For example
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)define a grid with x dimension from 1 to 9, y dimension from -3 to 5 and z dimension from 1 to 3. The grid is subdivided in 3 x 3 x 3 blocks which have the size 3 x 3 x 1 grid cells.
See also: module fd4_vartab_mod
public subroutine fd4_domain_init_metadata (domain, maxext, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (3) :: maxext
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_init_metadata
Parameters:| domain | the domain |
| maxext | max extent of couple arrays |
| err | error status: 0...ok |
This subroutine must be called before creating couple contexts for domain. If it is not called, no decomposed management is applied. Decomposed management is not implemented for adaptive block mode.
With maxext you can set the maximum extent of couple arrays in number of grid cells per dimension (without extension by face variables). If any value in maxext is <=0, decomposition is disabled.
public subroutine fd4_domain_adapt_metadata (domain)
type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_adapt_metadata
Parameters:| domain | the domain |
Once this subroutine has been called, the meta data will be optimized at each repartitioning. You must first initialize decomposed management of meta date with fd4_domain_init_metadata.
public subroutine fd4_domain_dump_stats (domain, opt_root, opt_out)
type (fd4_domain), intent(inout) :: domain
integer, optional, intent(in) :: opt_root
integer, optional, intent(in) :: opt_out
end subroutine fd4_domain_dump_stats
Parameters:| domain | the domain |
| opt_root | root MPI process that prints statistics |
| opt_out | unit for output |
public subroutine fd4_domain_clear (domain)
type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_clear
Parameters:| domain | the domain |
public subroutine fd4_domain_delete (domain, opt_stats)
type (fd4_domain), intent(inout) :: domain
logical, optional, intent(in) :: opt_stats
end subroutine fd4_domain_delete
Parameters:| domain | the domain |
| opt_stats | dump FD4 statistics to stdout? default is yes. |
Calls fd4_domain_dump_stats and deallocates all data associated with domain.