Dendro  5.01
Dendro in Greek language means tree. The Dendro library is a large scale (262K cores on ORNL's Titan) distributed memory adaptive octree framework. The main goal of Dendro is to perform large scale multiphysics simulations efficeiently in mordern supercomputers. Dendro consists of efficient parallel data structures and algorithms to perform variational ( finite element) methods and finite difference mthods on 2:1 balanced arbitary adaptive octrees which enables the users to perform simulations raning from black holes (binary black hole mergers) to blood flow in human body, where applications ranging from relativity, astrophysics to biomedical engineering.
Public Member Functions | Protected Attributes | List of all members
ot::subDA Class Reference

subDA based on the DA, where to perform computations sub domain of the main domain. More...

#include <sub_oda.h>

Collaboration diagram for ot::subDA:
Collaboration graph
[legend]

Public Member Functions

 subDA (DA *da, std::function< double(double, double, double) > fx_retain, double *gSize)
 : defult sub da constructor. More...
 
ot::DAglobal_domain ()
 
unsigned int getLocalNodalSz () const
 returns the local nodal size
 
unsigned int getPreNodalSz () const
 returns the pre ghost nodal size
 
unsigned int getPostNodalSz () const
 returns the post nodal size
 
unsigned int getTotalNodalSz () const
 returns the total nodal size (this includes the ghosted region as well.)
 
unsigned int getLocalElemSz () const
 returns the local elemental size
 
unsigned int getTotalElemSz () const
 returns the local elemental size (includes the ghost elements as well)
 
bool isActive () const
 see if the current DA is active
 
unsigned int getNumNodesPerElement () const
 get number of nodes per element
 
unsigned int getElementOrder () const
 get element order
 
MPI_Comm getGlobalComm () const
 : returns the global MPI communicator
 
const ot::MeshgetMesh () const
 : returns node local to node global map More...
 
MPI_Comm getCommActive () const
 : returns the active MPI sub com of the global communicator
 
unsigned int getNpesAll () const
 : global mpi com. size
 
unsigned int getNpesActive () const
 : number of processors active
 
unsigned int getRankAll () const
 : rank with respect to the global comm.
 
unsigned int getRankActive () const
 : rank w.r.t active comm.
 
bool isBoundaryOctant (unsigned int eleID) const
 : returns true if specified eleID is a boundary element false if the eleID is local and not a boundary element. More...
 
void getElementalCoords (unsigned int eleID, double *coords) const
 computes the elementCoordinates (based on the nodal placement) More...
 
const RefElementgetReferenceElement () const
 get a constant pointer for the reference element
 
unsigned int getElementSize () const
 : get the number of local elements.
 
unsigned int getPreGhostElementSize () const
 : get the number of pre-ghost element size
 
unsigned int getPostGhostElementSize () const
 : get the number of post-ghost element size
 
unsigned int getPreAndPostGhostNodeSize () const
 : get the number of pre and post ghost elements
 
unsigned int getGhostedElementSize () const
 : get the number of pre and post ghost elements
 
unsigned int getMaxDepth () const
 : get the max depth of the octree
 
unsigned int getDimension () const
 : get the dimensionality of the octree
 
template<typename T >
int createVector (T *&local, bool isElemental=false, bool isGhosted=false, unsigned int dof=1) const
 Creates a ODA vector. More...
 
template<typename T >
int createVector (std::vector< T > &local, bool isElemental=false, bool isGhosted=false, unsigned int dof=1) const
 Creates a ODA vector std::vector<T> More...
 
template<typename T >
void destroyVector (T *&local) const
 deallocates the memory allocated for a vector More...
 
template<typename T >
void destroyVector (std::vector< T > &local) const
 
template<ot::DA_FLAGS::LoopType type>
void init ()
 initialize the loop counters.
 
unsigned int curr ()
 get the current elmenent in the iteration
 
template<ot::DA_FLAGS::LoopType type>
unsigned int end ()
 Returns if the current element has reached end or not.
 
template<ot::DA_FLAGS::LoopType type>
void next ()
 : increment it to the next element.
 
template<typename T >
void readFromGhostBegin (T *vec, unsigned int dof=1)
 Initiate the ghost nodal value exchange.
 
template<typename T >
void readFromGhostEnd (T *vec, unsigned int dof=1)
 Sync the ghost element exchange.
 
template<typename T >
void writeToGhostsBegin (T *vec, unsigned int dof=1)
 Initiate accumilation across ghost elements.
 
template<typename T >
void writeToGhostsEnd (T *vec, unsigned int dof=1)
 Sync accumilation across ghost elements.
 
template<typename T >
void nodalVecToGhostedNodal (const T *in, T *&out, bool isAllocated=false, unsigned int dof=1) const
 convert nodal local vector with ghosted buffer regions. More...
 
template<typename T >
void ghostedNodalToNodalVec (const T *gVec, T *&local, bool isAllocated=false, unsigned int dof=1) const
 convert ghosted nodal vector to local vector (without ghosting) More...
 
template<typename T >
void getElementNodalValues (const T *in, T *eleVecOut, unsigned int eleID, unsigned int dof=1) const
 computes the element nodal values using interpolation if needed. More...
 
template<typename T >
void eleVecToVecAccumilation (T *out, const T *eleVecIn, unsigned int eleID, unsigned int dof=1) const
 computes the elemental vec to global vec accumilation More...
 
void getOctreeBoundaryNodeIndices (std::vector< unsigned int > &bdyIndex, std::vector< double > &coords, bool isGhosted=false)
 Computes the octree writable boundary nodes. More...
 
int getNodeIndices (DendroIntL *nodeIdx, unsigned int ele, bool isGhosted) const
 Returns the local indices to the nodes of the current element. More...
 
int getGlobalNodeIndices (DendroIntL *nodeIdx, unsigned int ele) const
 Returns the global node indices of a given element,. More...
 
template<typename T >
void setVectorByFunction (T *local, std::function< void(T, T, T, T *)>func, bool isElemental=false, bool isGhosted=false, unsigned int dof=1) const
 initialize a variable vector to a function depends on spatial coords. More...
 
template<typename T >
void setVectorByScalar (T *local, const T *value, bool isElemental=false, bool isGhosted=false, unsigned int dof=1) const
 initialize a variable vector to a function depends on spatial coords. More...
 
template<typename T >
void vecTopvtu (T *local, const char *fPrefix, char **nodalVarNames=NULL, bool isElemental=false, bool isGhosted=false, unsigned int dof=1)
 write the vec to pvtu file More...
 
unsigned int getLevel (unsigned int ele) const
 return the level of current octant More...
 
ot::TreeNode getOctant (unsigned int ele) const
 return the TreeNode of the current octnat. More...
 
template<typename T >
T * getVecPointerToDof (T *in, unsigned int dofInex, bool isElemental=false, bool isGhosted=false) const
 returns a pointer to a dof index, More...
 
template<typename T >
void copyVectors (T *dest, const T *source, bool isElemental=false, bool isGhosted=false, unsigned int dof=1) const
 copy vecotor to sorce to destination, assumes the same number of dof. More...
 
template<typename T >
void copyVector (T *dest, const T *source, bool isElemental=false, bool isGhosted=false) const
 more premitive copy, from source pointer to the dest pointer More...
 
ot::DAremesh (const DA_FLAGS::Refine *flags, unsigned int sz, unsigned int grainSz=100, double ld_bal=0.3, unsigned int sfK=2) const
 : Performs remesh based on the DA_FLAGS::Refine, which specifies no change, refine or coarsen. More...
 
template<typename T >
void intergridTransfer (const T *varIn, T *&varOut, const ot::DA *newDA, bool isElemental=false, bool isGhosted=false, unsigned int dof=1)
 performs grid transfer operations after the remesh. More...
 
template<typename T >
int getFaceNeighborValues (unsigned int eleID, const T *in, T *out, T *coords, unsigned int *neighID, unsigned int face, NeighbourLevel &level) const
 computes the face neighbor points for additional computations for a specified direction. More...
 

Protected Attributes

ot::DAm_da
 : parent da
 
std::vector< unsigned char > m_ucpSkipList
 :elemental skip list
 
std::vector< unsigned char > m_ucpSkipNodeList
 :nodal skip list
 
unsigned int m_uiLocalElementSize
 : local element size
 
unsigned int m_uiPreGhostElementSize
 : pre ghost element size
 
unsigned int m_uiPostGhostElementSize
 : post ghost element size
 
unsigned int m_uiLocalNodeSize
 : Local node size
 
unsigned int m_uiPreGhostNodeSize
 : pre ghost node size
 
unsigned int m_uiPostGhostNodeSize
 : post ghost node size
 
unsigned int m_uiCommTag
 : async comm tag
 
std::vector< AsyncExchangeContexm_uiMPIContexts
 
LoopCounter m_uiLoopInfo
 : Loop counter info
 
std::vector< AsyncExchangeContexm_mpiContexts
 : async comm contex
 
std::vector< unsigned int > m_uip_sub2DA_ElemMap
 : subDA to DA map (elemental)
 
std::vector< unsigned int > m_uip_DA2sub_ElemMap
 : DA to subDA map (elemental)
 
std::vector< unsigned int > m_uip_sub2DA_NodeMap
 : subDA to DA map (nodal)
 
std::vector< unsigned int > m_uip_DA2sub_NodeMap
 : DA to subDA map (nodal)
 
double m_dMinBB [3]
 min bound box for fe_retain
 
double m_dMaxBB [3]
 max bound box for fe_retain
 
unsigned int m_uiTotalNodalSz
 total nodal size (zipped nodal size)
 
unsigned int m_uiLocalNodalSz
 number of local nodes
 
unsigned int m_uiTotalElementSz
 total element size
 
unsigned int m_uiLocalElementSz
 local elemental size
 
unsigned int m_uiElementPreGhostBegin = 0
 
unsigned int m_uiElementPreGhostEnd
 
unsigned int m_uiElementLocalBegin
 
unsigned int m_uiElementLocalEnd
 
unsigned int m_uiElementPostGhostBegin
 
unsigned int m_uiElementPostGhostEnd
 
unsigned int m_uiNodePreGhostBegin =0
 
unsigned int m_uiNodePreGhostEnd
 
unsigned int m_uiNodeLocalBegin
 
unsigned int m_uiNodeLocalEnd
 
unsigned int m_uiNodePostGhostBegin
 
unsigned int m_uiNodePostGhostEnd
 
unsigned int m_uiNumPreGhostElements
 
unsigned int m_uiNumLocalElements
 
unsigned int m_uiNumPostGhostElements
 
unsigned int m_uiNumPreGhostNodes
 
unsigned int m_uiNumLocalNodes
 
unsigned int m_uiNumPostGhostNodes
 
std::vector< unsigned int > m_uiSendScatterMap
 : Send scatter map
 
std::vector< unsigned int > m_uiRecvScatterMap
 : recv scatter map
 
std::vector< unsigned int > m_uiSendCounts
 : send counts
 
std::vector< unsigned int > m_uiRecvCounts
 : recv counts
 
std::vector< unsigned int > m_uiSendOffsets
 : send offsets
 
std::vector< unsigned int > m_uiRecvOffsets
 : recv offsets
 
std::vector< unsigned int > m_uiSendProcList
 
std::vector< unsigned int > m_uiRecvProcList
 
bool m_uiIsActive
 : active sub da
 
MPI_Comm m_uiCommGlobal
 : MPI comm global
 
MPI_Comm m_uiCommActive
 : MPI_Comm for the active
 
int m_uiRankActive
 : rank active
 
int m_uiRankGlobal
 : rank global
 
int m_uiNpesActive
 : npes active
 
int m_uiNpesGlobal
 : npes global
 

Detailed Description

subDA based on the DA, where to perform computations sub domain of the main domain.

Constructor & Destructor Documentation

◆ subDA()

ot::subDA::subDA ( ot::DA da,
std::function< double(double, double, double) >  fx_retain,
double *  gSize 
)

: defult sub da constructor.

Parameters
[in]dainput DA
[in]fx_retain: area to retain param[in] gSize: global domain size.

Member Function Documentation

◆ copyVector()

template<typename T >
void ot::subDA::copyVector ( T *  dest,
const T *  source,
bool  isElemental = false,
bool  isGhosted = false 
) const

more premitive copy, from source pointer to the dest pointer

Parameters
[in/out]dest: destination pointer
[in]sourcesource pointer
[in]isElementaltrue if this is an elemental vector/ false otherwise
[in]isGhostedtrue if this is a ghosted vector

◆ copyVectors()

template<typename T >
void ot::subDA::copyVectors ( T *  dest,
const T *  source,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
) const

copy vecotor to sorce to destination, assumes the same number of dof.

Parameters
[in/out]dest: destination pointer
[in]sourcesource pointer
[in]isElementaltrue if this is an elemental vector/ false otherwise
[in]isGhostedtrue if this is a ghosted vector
[in]dofdegrees of freedoms

◆ createVector() [1/2]

template<typename T >
int ot::subDA::createVector ( T *&  local,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
) const

Creates a ODA vector.

Parameters
[in]local: VecType pointer
[in]isElementalTrue if creating a elemental vector (cell data vector) false for a nodal vector
[in]isGhostedTrue will allocate ghost nodal values as well, false will only allocate memory for local nodes.
[in]dofdegrees of freedoms

◆ createVector() [2/2]

template<typename T >
int ot::subDA::createVector ( std::vector< T > &  local,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
) const

Creates a ODA vector std::vector<T>

Parameters
[in]local: VecType pointer
[in]isElementalTrue if creating a elemental vector (cell data vector) false for a nodal vector
[in]isGhostedTrue will allocate ghost nodal values as well, false will only allocate memory for local nodes.
[in]dofdegrees of freedoms

◆ destroyVector()

template<typename T >
void ot::subDA::destroyVector ( T *&  local) const

deallocates the memory allocated for a vector

Parameters

◆ eleVecToVecAccumilation()

template<typename T >
void ot::subDA::eleVecToVecAccumilation ( T *  out,
const T *  eleVecIn,
unsigned int  eleID,
unsigned int  dof = 1 
) const

computes the elemental vec to global vec accumilation

Parameters
[out]outoutput (need to be ghosted )
[in]eleVecIneleVec values.
[in]eleIDelement ID
[in]dofdegrees of freedoms

◆ getElementalCoords()

void ot::subDA::getElementalCoords ( unsigned int  eleID,
double *  coords 
) const

computes the elementCoordinates (based on the nodal placement)

Parameters
[in]eleID: element ID

◆ getElementNodalValues()

template<typename T >
void ot::subDA::getElementNodalValues ( const T *  in,
T *  eleVecOut,
unsigned int  eleID,
unsigned int  dof = 1 
) const

computes the element nodal values using interpolation if needed.

Parameters
[in]ininput vector (need to be ghosted )

◆ getFaceNeighborValues()

template<typename T >
int ot::subDA::getFaceNeighborValues ( unsigned int  eleID,
const T *  in,
T *  out,
T *  coords,
unsigned int *  neighID,
unsigned int  face,
NeighbourLevel &  level 
) const

computes the face neighbor points for additional computations for a specified direction.

Parameters
[in]eleIDelement ID
[in]ininpute vector
[out]outoutput vector values are in the order of the x,y,z size : 4*NodesPerElement
[out]coordsget the corresponding coordinates size: 4*NodesPerElement*m_uiDim;
[out]neighIDface neighbor octant IDs,
[in]faceface direction in {OCT_DIR_LEFT,OCT_IDR_RIGHT,OCT_DIR_DOWN, OCT_DIR_UP,OCT_DIR_BACK,OCT_DIR_FRONT}
[out]levelthe level of the neighbour octant with respect to the current octant. returns the number of face neighbours 1/4 for 3D.

◆ getGlobalNodeIndices()

int ot::subDA::getGlobalNodeIndices ( DendroIntL *  nodeIdx,
unsigned int  ele 
) const

Returns the global node indices of a given element,.

Parameters
[out]nodeIdxglobal node indices,
[in]eleelement id

◆ getLevel()

unsigned int ot::subDA::getLevel ( unsigned int  ele) const
inline

return the level of current octant

Parameters
[in]eleelementID of the current octant

◆ getMesh()

const ot::Mesh* ot::subDA::getMesh ( ) const
inline

: returns node local to node global map

returns the mesh

◆ getNodeIndices()

int ot::subDA::getNodeIndices ( DendroIntL *  nodeIdx,
unsigned int  ele,
bool  isGhosted 
) const

Returns the local indices to the nodes of the current element.

Parameters
nodesIndices into the nodes of the given element. Should be allocated by the user prior to calling.
Returns
-1 for error

◆ getOctant()

ot::TreeNode ot::subDA::getOctant ( unsigned int  ele) const
inline

return the TreeNode of the current octnat.

Parameters
[in]eleelementID of the current octant

◆ getOctreeBoundaryNodeIndices()

void ot::subDA::getOctreeBoundaryNodeIndices ( std::vector< unsigned int > &  bdyIndex,
std::vector< double > &  coords,
bool  isGhosted = false 
)

Computes the octree writable boundary nodes.

Parameters

◆ getVecPointerToDof()

template<typename T >
T* ot::subDA::getVecPointerToDof ( T *  in,
unsigned int  dofInex,
bool  isElemental = false,
bool  isGhosted = false 
) const

returns a pointer to a dof index,

Parameters
[in]ininput vector pointer
[in]dofInexdof index which is the pointer is needed, should be less than dof, value the vector created.
[in]isElementaltrue if this is an elemental vector/ false otherwise
[in]isGhostedtrue if this is a ghosted vector
Returns
pointer to dofIndex.

◆ ghostedNodalToNodalVec()

template<typename T >
void ot::subDA::ghostedNodalToNodalVec ( const T *  gVec,
T *&  local,
bool  isAllocated = false,
unsigned int  dof = 1 
) const

convert ghosted nodal vector to local vector (without ghosting)

Parameters
[in]gVecghosted vector
[out]locallocal vector (assume an allocated vector)
[in]isAllocatedtrue if the out is allocated, false otherwise.
[in]dofdegrees of freedoms

◆ intergridTransfer()

template<typename T >
void ot::subDA::intergridTransfer ( const T *  varIn,
T *&  varOut,
const ot::DA newDA,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
)

performs grid transfer operations after the remesh.

Parameters
[in]varInvariable defined by oldDA
[out]varOutvariable defined by newDA. interpolate varOut from varIn. (Note: varOut allocated inside the function, no need to allocate outside)
[in]isElementaltrue if it is an elemental vector
[in]isGhostedtrue if allocated ghost vector
[in]dofdegrees of freedoms.

◆ isBoundaryOctant()

bool ot::subDA::isBoundaryOctant ( unsigned int  eleID) const

: returns true if specified eleID is a boundary element false if the eleID is local and not a boundary element.

Parameters
[in]eleIDelement ID

◆ nodalVecToGhostedNodal()

template<typename T >
void ot::subDA::nodalVecToGhostedNodal ( const T *  in,
T *&  out,
bool  isAllocated = false,
unsigned int  dof = 1 
) const

convert nodal local vector with ghosted buffer regions.

Parameters
[in]ininput vector (should be nodal and non ghosted)
[out]outcoverted nodal vector with ghost regions.
[in]isAllocatedtrue if the out is allocated, false otherwise.
[in]dofdegrees of freedoms

◆ remesh()

ot::DA* ot::subDA::remesh ( const DA_FLAGS::Refine flags,
unsigned int  sz,
unsigned int  grainSz = 100,
double  ld_bal = 0.3,
unsigned int  sfK = 2 
) const

: Performs remesh based on the DA_FLAGS::Refine, which specifies no change, refine or coarsen.

Parameters
[in]flagsrefinement flags.
[in]szsize of the array flags (needs to be size of the local elements)
[in]grainSzrougly the number of octants per core you need when you create the new da.
[in]ld_tolload imbalance tolerance.
[in]sfKsplitter fix factor. better to be power of two. increase the value to 128 when running on > 64,000 cores
Returns
: Specifies the new grid, with new DA.

◆ setVectorByFunction()

template<typename T >
void ot::subDA::setVectorByFunction ( T *  local,
std::function< void(T, T, T, T *)>  func,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
) const

initialize a variable vector to a function depends on spatial coords.

Parameters

◆ setVectorByScalar()

template<typename T >
void ot::subDA::setVectorByScalar ( T *  local,
const T *  value,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
) const

initialize a variable vector to a function depends on spatial coords.

Parameters

◆ vecTopvtu()

template<typename T >
void ot::subDA::vecTopvtu ( T *  local,
const char *  fPrefix,
char **  nodalVarNames = NULL,
bool  isElemental = false,
bool  isGhosted = false,
unsigned int  dof = 1 
)

write the vec to pvtu file

Parameters
[in]localvariable vector
[in]fPrefixfile name prefix
[in]isElementalTrue if creating a elemental vector (cell data vector) false for a nodal vector
[in]isGhostedTrue will allocate ghost nodal values as well, false will only allocate memory for local nodes.
[in]dofdegrees of freedoms

Member Data Documentation

◆ m_uiElementLocalBegin

unsigned int ot::subDA::m_uiElementLocalBegin
protected

begin of locat octants (INDEPENDENT)

◆ m_uiElementLocalEnd

unsigned int ot::subDA::m_uiElementLocalEnd
protected

end of the local octants (INDEPENDET)

◆ m_uiElementPostGhostBegin

unsigned int ot::subDA::m_uiElementPostGhostBegin
protected

begin location for the post ghost octants

◆ m_uiElementPostGhostEnd

unsigned int ot::subDA::m_uiElementPostGhostEnd
protected

end location for the post ghost octants

◆ m_uiElementPreGhostBegin

unsigned int ot::subDA::m_uiElementPreGhostBegin = 0
protected

begin of the pre ghost elements

◆ m_uiElementPreGhostEnd

unsigned int ot::subDA::m_uiElementPreGhostEnd
protected

end of pre ghost elements

◆ m_uiNodeLocalBegin

unsigned int ot::subDA::m_uiNodeLocalBegin
protected

begin location of the local nodes in CG indexing

◆ m_uiNodeLocalEnd

unsigned int ot::subDA::m_uiNodeLocalEnd
protected

end location of the local nodes in CG indexing

◆ m_uiNodePostGhostBegin

unsigned int ot::subDA::m_uiNodePostGhostBegin
protected

begin location of the post ghost nodes in CG indexing

◆ m_uiNodePostGhostEnd

unsigned int ot::subDA::m_uiNodePostGhostEnd
protected

end location of the post ghost nodes in CG indexing

◆ m_uiNodePreGhostBegin

unsigned int ot::subDA::m_uiNodePreGhostBegin =0
protected

begin location of the pre ghost nodes in CG indexing

◆ m_uiNodePreGhostEnd

unsigned int ot::subDA::m_uiNodePreGhostEnd
protected

end location of the pre ghost nodes in CG indexing

◆ m_uiNumLocalElements

unsigned int ot::subDA::m_uiNumLocalElements
protected

number of local elements

◆ m_uiNumLocalNodes

unsigned int ot::subDA::m_uiNumLocalNodes
protected

number of local nodes

◆ m_uiNumPostGhostElements

unsigned int ot::subDA::m_uiNumPostGhostElements
protected

number of post ghost elements

◆ m_uiNumPostGhostNodes

unsigned int ot::subDA::m_uiNumPostGhostNodes
protected

number of post nodes

◆ m_uiNumPreGhostElements

unsigned int ot::subDA::m_uiNumPreGhostElements
protected

number of pre ghost elements

◆ m_uiNumPreGhostNodes

unsigned int ot::subDA::m_uiNumPreGhostNodes
protected

number of pre ghost nodes

◆ m_uiRecvProcList

std::vector<unsigned int> ot::subDA::m_uiRecvProcList
protected

recv processor list for the ghost exchange

◆ m_uiSendProcList

std::vector<unsigned int> ot::subDA::m_uiSendProcList
protected

Send processor list for ghost exchange


The documentation for this class was generated from the following files: