module fd4_block_mod
module fd4_globaldef_mod
module fd4_vartab_mod
module fd4_ghostlist_mod
module fd4_mpi_mod
module rbtree_fd4_block_mod
module rbtree_int8_mod
module stack_mod
module stack3_mod
module section_mod
Types:
public type fd4_domain_balance
public type fd4_domain_coord
public type fd4_domain_blockpool
public type fd4_domain
Variables:
integer, public, parameter :: c2i_shft3 = 20
integer, public, parameter :: c2i_shft2 = 10
integer, public, parameter :: FD4_LB_HISTORY = 4
Interfaces:
public interface fd4_domain_create
Subroutines and functions:
public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc)
public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc)
public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc)
public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc)
public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
public subroutine fd4_domain_clear (domain)
public subroutine fd4_domain_delete (domain)
Naming conventions for spatial 3D position, size, and bounds of a block, domain, etc.:
Author: Matthias Lieber
public type fd4_domain_balance
real (kind=r4k) :: defaultweight = 1.0_r4k
real (kind=r4k), pointer, dimension (:,:,:) :: blkweights
real (kind=r4k) :: localweight
integer (kind=i_k), pointer, dimension (:) :: hidx => NULL ()
integer (kind=i_k), pointer, dimension (:) :: rnk => NULL ()
type (stack3) :: allblocks
integer (kind=i_k) :: last_sfctype = -1
real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbtimes
integer (kind=i_k) :: curlbtime
logical :: globalize_lbtime
real (kind=r4k), dimension (FD4_LB_HISTORY) :: lbbalance
integer (kind=i_k) :: curlbbalance
logical :: part_changed
end type fd4_domain_balance
Components:| defaultweight | default weight of new created blocks (per grid cell!) |
| blkweights | weights of all blocks |
| localweight | sum of local block weights, updated in fd4_part_sfc |
| hidx | hilbert sfc index of global allocated blocks |
| rnk | ranking of global allocated blocks by hilbert sfc index |
| allblocks | coordinates of all required blocks in global domain |
| last_sfctype | last used type of SFC curve |
| lbtimes | timings of last FD4_LB_HISTORY calls of load balancing |
| curlbtime | index of current entry in lbtimes |
| globalize_lbtime | indicate whether current lbtime needs to be globalized (max) |
| lbbalance | balance achieved by last FD4_LB_HISTORY calls of load balancing |
| curlbbalance | index of current entry in lbbalance |
| part_changed | indicate whether fd4_balance_readjust tried to balance the load last time |
TODO: put other domain stuff here, e.g. empties, hilbert_level
public type fd4_domain_coord
integer (kind=i_k), pointer, dimension (:) :: bl => null ()
end type fd4_domain_coord
Components:| bl | start indices of blocks |
public type fd4_domain_blockpool
type (fd4_block_link), pointer, dimension (:) :: stack => null ()
integer (kind=i_k) :: pos = 0
end type fd4_domain_blockpool
A pool of unused blocks, implemented a fixed size stack. Internally used by fd4_domain.
public type fd4_domain
type (rbtree_fd4_block) :: blocks
type (fd4_domain_coord), dimension (3) :: d
integer (kind=i_k), dimension (3,2) :: dbnd
integer (kind=i_k), dimension (3) :: blnum
integer (kind=i_k), dimension (3,2) :: mmbsz
integer (kind=i_k), dimension (3) :: nghosts
logical, dimension (3) :: periodic
type (fd4_vartab), pointer, dimension (:) :: vartab => null ()
type (fd4_varinfo) :: varinfo
type (fd4_ghostlist), dimension (-1:0) :: glist
type (fd4_mpi) :: mpi
integer, pointer, dimension (:,:,:,:) :: blkmap => null ()
integer :: blkmap_now
integer :: blkmap_last
logical :: init = .false.
integer (kind=i_k) :: hilbert_level
integer (kind=i8k) :: partition_id
integer (kind=i4k), pointer, dimension (:,:,:) :: empties
integer (kind=i_k) :: blockcount = 0
type (section) :: blockunion
type (fd4_domain_balance) :: balance
type (stack) :: markedblocks
logical :: blocks_are_marked
type (fd4_domain_blockpool), pointer, dimension (:,:,:) :: pool
type (rbtree_int8) :: cpltypestore
integer (kind=i4k), dimension (14) :: cplblkenc
end type fd4_domain
Components:| blocks | data structure containing all blocks of the domain |
| d | information about each of the 3 spatial dimensions |
| dbnd | start and end indices of the grid for x,y,z coordinates |
| blnum | number of blocks in x,y,z direction |
| mmbsz | min and max block size in each spatial dimension |
| nghosts | number of ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| vartab | variable table |
| varinfo | global info about variable table |
| glist | lists of all ghost blocks (boundary + communication ghosts) |
| mpi | MPI context |
| blkmap | Who owns which block? Contains MPI ranks. With boundary. |
| blkmap_now | current block map (4th dimension of blkmap) |
| blkmap_last | last block map (4th dimension of blkmap) |
| init | true if domain is initialized |
| hilbert_level | level of the Hilbert SFC (depends on blnum) |
| partition_id | unique identifier of the current partition (counter) |
| empties | block%sempty of all blocks |
| blockcount | global number of blocks |
| blockunion | spatial union of bounds of all local blocks |
| balance | Stores data related to dynamic load balancing |
| markedblocks | list of blocks (idx) marked as to be created (global) |
| blocks_are_marked | are there any blocks marked (global) |
| pool | block pools for each possible block size |
| cpltypestore | a binary tree to store MPI types for coupling |
| cplblkenc | encoding params of index for this tree |
See also: module fd4_vartab_mod, module rbtree_fd4_block_mod
public interface fd4_domain_create
module procedure fd4_domain_create_fixed
module procedure fd4_domain_create_specific
end interface fd4_domain_create
Create new domain.
public subroutine fd4_domain_add_block (domain, bpos, init, err, bl, opt_alloc, opt_dealloc)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (3) :: bpos
logical, intent(in) :: init
integer (kind=i_k), intent(out) :: err
type (fd4_block), optional, pointer :: bl
integer (kind=i_k), optional, intent(inout) :: opt_alloc
integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_add_block
Parameters:| domain | the domain |
| bpos | position of the block in the block decomposition |
| init | if true, data fields are initialized with their vnull |
| err | error status: 0...ok, -1...block already present, >0...error |
| bl | optionally return a pointer to the new block |
| opt_alloc | add here the number of real block allocations (no block in pool found) |
| opt_dealloc | add here the number of real block deallocations |
Allocates new block and block's static arrays. Adds the block to the block data structure. Set up the neighbor pointers of this and neighbor blocks. The value for bpos(i) must be >=1 and <=domain%blnum(i).
public subroutine fd4_domain_add_ghost (domain, b, dir, dim, gtype, opt_alloc)
type (fd4_domain), intent(inout) :: domain
type (fd4_block), pointer :: b
integer (kind=i_k), intent(in) :: dir
integer (kind=i_k), intent(in) :: dim
integer (kind=i_k), intent(in) :: gtype
integer (kind=i_k), optional, intent(inout) :: opt_alloc
end subroutine fd4_domain_add_ghost
Parameters:| domain | the domain |
| b | the block |
| dir | boundary ghost block direction |
| dim | boundary ghost block dimension |
| gtype | the type of ghost (FD4_GHOST_BND or FD4_GHOST_CMM) |
| opt_alloc | add here the number of real block allocations (no block in pool found) |
Note: Data fields of ghost blocks are not initialized with vnull, since is is expected that these data is read only after ghost communication.
Called from fd4_domain_add_block and in fd4_balance_readjust. The new gost block bgb is added to the ghostlist glist and a pointer from bgb is set to b:
bgb%neigh(2,1)%l => b
Here is a list of all block components which are abused by ghost blocks:
bgb%neigh(1,1)%l ... previous in ghostlist bgb%neigh(1,2)%l ... next in ghostlist bgb%neigh(2,1)%l ... pointer to neighor within domain (ordinary block), "parent" bgb%pos(1) ... boundary ghost block indicator, (FD4_GHOST_BND or FD4_GHOST_CMM) bgb%pos(2) ... boundary direction (1 or 2) bgb%pos(3) ... boundary dimension (1, 2 or 3)
TODO:
public subroutine fd4_domain_delete_block (domain, hidx, err, opt_dealloc)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in) :: hidx
integer (kind=i_k), intent(out) :: err
integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_delete_block
Parameters:| domain | the domain |
| hidx | the index of the block on the SFC |
| err | error status: 0...ok |
| opt_dealloc | add here the number of real block deallocations (no space left in pool) |
Deletes all memory associated with the block, including both types of ghost blocks. Clears neighbor pointers in neighbor blocks.
public subroutine fd4_domain_delete_ghost (domain, gb, opt_dealloc)
type (fd4_domain), intent(inout) :: domain
type (fd4_block), pointer :: gb
integer (kind=i_k), optional, intent(inout) :: opt_dealloc
end subroutine fd4_domain_delete_ghost
Parameters:| domain | the domain |
| gb | the ghost block |
| opt_dealloc | add here the number of real block deallocations (no space left in pool) |
public subroutine fd4_domain_max_bext (domain, bext, add_ghosts)
type (fd4_domain), intent(in) :: domain
integer (kind=i_k), intent(out), dimension (3) :: bext
logical, optional, intent(in) :: add_ghosts
end subroutine fd4_domain_max_bext
Parameters:| domain | the domain |
| bext | block extents greater than or equal to any block's extents, 0 on error |
| add_ghosts | if present and true, adds ghost cells to the extent |
This is not necessarily the extent of the largest block. The allocation status of the blocks does not influence the result of this routine.
private subroutine fd4_domain_create_fixed (domain, blnum, dbnd, vars, nghosts, periodic, mpi_comm, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (3) :: blnum
integer (kind=i_k), intent(in), dimension (3,2) :: dbnd
type (fd4_vartab), intent(in), dimension (:) :: vars
integer (kind=i_k), intent(in), dimension (3) :: nghosts
logical, intent(in), dimension (3) :: periodic
integer (kind=i_k), intent(in) :: mpi_comm
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_fixed
Parameters:| domain | the domain |
| blnum | number of blocks in x,y,z direction |
| dbnd | start and end indices of the grid for x,y,z coordinates |
| vars | variable table |
| nghosts | number of required ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| mpi_comm | MPI communicator |
| err | error status: 0...ok |
This function calculates the actual block bounds from the number of blocks in each spatial direction and the start and end indices of the grid.
For example, the values
dbnd(:,1) = (/ 1,-3, 1/) ! start indices of grid in x,y,z direction dbnd(:,2) = (/ 9, 5, 3/) ! end indices of grid in x,y,z direction blnum = (/ 3, 3, 3/) ! number of blocks in x,y,z directionresult in the following block definitions used for fd4_domain_create_specific:
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)
See also: module fd4_vartab_mod
private subroutine fd4_domain_create_specific (domain, blx, bly, blz, vars, nghosts, periodic, mpi_comm, err)
type (fd4_domain), intent(inout) :: domain
integer (kind=i_k), intent(in), dimension (:) :: blx
integer (kind=i_k), intent(in), dimension (:) :: bly
integer (kind=i_k), intent(in), dimension (:) :: blz
type (fd4_vartab), intent(in), dimension (:) :: vars
integer (kind=i_k), intent(in), dimension (3) :: nghosts
logical, intent(in), dimension (3) :: periodic
integer (kind=i_k), intent(in) :: mpi_comm
integer (kind=i_k), intent(out) :: err
end subroutine fd4_domain_create_specific
Parameters:| domain | the domain |
| blx | start indices of blocks in x direction |
| bly | start indices of blocks in y direction |
| blz | start indices of blocks in z direction |
| vars | variable table |
| nghosts | number of required ghost cells for each spatial dimension |
| periodic | apply periodic boundary conditions? |
| mpi_comm | MPI communicator |
| err | error status: 0...ok |
The array blx is of length num_x_blocks + 1. The first num_x_blocks values define the start indices of the blocks in x direction. The last value defines the start index of an additional imaginary block and thus the size of the last real block. The same applies for bly and blz. The bounds of the grid is derived from the block definitions. For example
blx = (/ 1, 4, 7, 10/) bly = (/-3, 0, 3, 6/) blz = (/ 1, 2, 3, 4/)define a grid with x dimension from 1 to 9, y dimension from -3 to 5 and z dimension from 1 to 3. The grid is subdivided in 3 x 3 x 3 blocks which have the size 3 x 3 x 1 grid cells.
See also: module fd4_vartab_mod
public subroutine fd4_domain_clear (domain)
type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_clear
Parameters:| domain | the domain |
public subroutine fd4_domain_delete (domain)
type (fd4_domain), intent(inout) :: domain
end subroutine fd4_domain_delete
Parameters:| domain | the domain |
Deallocates all data associated with domain.