27 #ifndef EWOMS_DOMESTIC_OVERLAP_FROM_BCRS_MATRIX_HH
28 #define EWOMS_DOMESTIC_OVERLAP_FROM_BCRS_MATRIX_HH
65 template <
class BCRSMatrix>
67 const BorderList& borderList,
70 : foreignOverlap_(A, borderList, blackList, overlapSize)
71 , blackList_(blackList)
72 , globalIndices_(foreignOverlap_)
79 MPI_Comm_rank(MPI_COMM_WORLD, &tmp);
80 myRank_ =
static_cast<ProcessRank
>(tmp);
81 MPI_Comm_size(MPI_COMM_WORLD, &tmp);
82 worldSize_ =
static_cast<unsigned>(tmp);
85 buildDomesticOverlap_();
87 blackList_.updateNativeToDomesticMap(*
this);
96 for (
unsigned domIdx = 0; domIdx <
numDomestic(); ++domIdx) {
103 std::map<int, MpiBuffer<unsigned> *> sizeBufferMap;
105 auto peerIt = peerSet_.begin();
106 const auto& peerEndIt = peerSet_.end();
107 for (; peerIt != peerEndIt; ++peerIt) {
109 sizeBufferMap[*peerIt] = &buffer;
111 buffer.send(*peerIt);
114 peerIt = peerSet_.begin();
115 for (; peerIt != peerEndIt; ++peerIt) {
116 MpiBuffer<unsigned> rcvBuffer(1);
117 rcvBuffer.receive(*peerIt);
119 assert(rcvBuffer[0] == domesticOverlapWithPeer_.find(*peerIt)->second.size());
122 peerIt = peerSet_.begin();
123 for (; peerIt != peerEndIt; ++peerIt) {
124 sizeBufferMap[*peerIt]->wait();
125 delete sizeBufferMap[*peerIt];
139 {
return worldSize_; }
154 && foreignOverlap_.
isBorder(mapExternalToInternal_(domesticIdx));
164 && foreignOverlap_.
isBorderWith(mapExternalToInternal_(domesticIdx),
173 {
return foreignOverlap_.
numFront(peerRank); }
182 Index internalDomesticIdx = mapExternalToInternal_(domesticIdx);
186 const auto& domOverlap = domesticOverlapByIndex_[internalDomesticIdx];
187 return domOverlap.size() > 0
188 && domOverlap.begin()->second == foreignOverlap_.
overlapSize();
195 {
return blackList_; }
202 {
return domesticOverlapByIndex_[mapExternalToInternal_(domesticIdx)].size(); }
227 {
return foreignOverlap_.
numLocal(); }
246 {
return mapExternalToInternal_(domesticIdx) <
static_cast<Index
>(
numLocal()); }
256 return foreignOverlap_.
iAmMasterOf(mapExternalToInternal_(domesticIdx));
263 {
return masterRank_[
static_cast<unsigned>(mapExternalToInternal_(domesticIdx))]; }
269 { globalIndices_.
print(); }
279 return mapInternalToExternal_(internalIdx);
293 Index internalIdx = mapExternalToInternal_(domIdx);
294 if (internalIdx >= static_cast<Index>(
numLocal()))
307 return mapInternalToExternal_(localIdx);
316 return !this->
isLocal(domesticIdx)
317 || this->foreignOverlap_.
isInOverlap(mapExternalToInternal_(domesticIdx));
324 bool isFrontFor(ProcessRank peerRank, Index domesticIdx)
const
326 Index internalIdx = mapExternalToInternal_(domesticIdx);
327 return this->foreignOverlap_.
isFrontFor(peerRank, internalIdx);
336 mapExternalToInternal_(domesticIdx));
355 return mapInternalToExternal_(internalIdx);
363 {
return domesticOverlapWithPeer_.at(peerRank).size(); }
372 Index internalIdx = domesticOverlapWithPeer_.at(peerRank)[overlapOffset];
373 return mapInternalToExternal_(internalIdx);
377 void buildDomesticOverlap_()
380 peerSet_ = foreignOverlap_.
peerSet();
384 domesticOverlapByIndex_.resize(
numLocal());
385 borderDistance_.resize(
numLocal(), 0);
387 PeerSet::const_iterator peerIt;
388 PeerSet::const_iterator peerEndIt = peerSet_.end();
391 peerIt = peerSet_.begin();
392 for (; peerIt != peerEndIt; ++peerIt) {
393 ProcessRank peerRank = *peerIt;
394 sendIndicesToPeer_(peerRank);
398 peerIt = peerSet_.begin();
399 for (; peerIt != peerEndIt; ++peerIt) {
400 ProcessRank peerRank = *peerIt;
401 receiveIndicesFromPeer_(peerRank);
405 peerIt = peerSet_.begin();
406 for (; peerIt != peerEndIt; ++peerIt) {
407 ProcessRank peerRank = *peerIt;
408 waitSendIndices_(peerRank);
412 void updateMasterRanks_()
416 masterRank_.resize(nDomestic);
420 for (
unsigned i = 0; i < nLocal; ++i) {
421 masterRank_[i] = foreignOverlap_.
masterRank(static_cast<Index>(i));
426 for (
size_t i = nLocal; i < nDomestic; ++i)
427 masterRank_[i] = std::numeric_limits<ProcessRank>::max();
431 auto peerIt = peerSet_.begin();
432 const auto& peerEndIt = peerSet_.end();
433 for (; peerIt != peerEndIt; ++peerIt) {
434 const auto& overlapWithPeer = domesticOverlapWithPeer_.find(*peerIt)->second;
436 auto idxIt = overlapWithPeer.begin();
437 const auto& idxEndIt = overlapWithPeer.end();
438 for (; idxIt != idxEndIt; ++idxIt) {
439 if (*idxIt >= 0 && foreignOverlap_.
isLocal(*idxIt))
442 masterRank_[
static_cast<unsigned>(*idxIt)] = std::min(masterRank_[static_cast<unsigned>(*idxIt)], *peerIt);
447 void sendIndicesToPeer_(ProcessRank peerRank)
455 size_t numIndices = foreignOverlap.size();
456 numIndicesSendBuffer_[peerRank] =
new MpiBuffer<size_t>(1);
457 (*numIndicesSendBuffer_[peerRank])[0] = numIndices;
458 numIndicesSendBuffer_[peerRank]->send(peerRank);
461 indicesSendBuffer_[peerRank] =
new MpiBuffer<IndexDistanceNpeers>(numIndices);
464 auto overlapIt = foreignOverlap.begin();
465 const auto& overlapEndIt = foreignOverlap.end();
466 for (
unsigned i = 0; overlapIt != overlapEndIt; ++overlapIt, ++i) {
467 Index localIdx = overlapIt->index;
468 BorderDistance borderDistance = overlapIt->borderDistance;
471 IndexDistanceNpeers tmp;
473 tmp.borderDistance = borderDistance;
474 tmp.numPeers =
static_cast<unsigned>(
numPeers);
476 (*indicesSendBuffer_[peerRank])[i] = tmp;
479 indicesSendBuffer_[peerRank]->send(peerRank);
483 void waitSendIndices_(ProcessRank peerRank)
485 numIndicesSendBuffer_[peerRank]->wait();
486 delete numIndicesSendBuffer_[peerRank];
488 indicesSendBuffer_[peerRank]->wait();
489 delete indicesSendBuffer_[peerRank];
492 void receiveIndicesFromPeer_(ProcessRank peerRank)
497 MpiBuffer<size_t> numIndicesRecvBuff(1);
498 numIndicesRecvBuff.receive(peerRank);
499 numIndices =
static_cast<int>(numIndicesRecvBuff[0]);
502 MpiBuffer<IndexDistanceNpeers> recvBuff(static_cast<size_t>(numIndices));
503 recvBuff.receive(peerRank);
504 for (
unsigned i = 0; i < static_cast<unsigned>(numIndices); ++i) {
505 Index globalIdx = recvBuff[i].index;
506 BorderDistance borderDistance = recvBuff[i].borderDistance;
511 Index newDomesticIdx =
static_cast<Index
>(globalIndices_.
numDomestic());
512 globalIndices_.
addIndex(newDomesticIdx, globalIdx);
515 borderDistance_.resize(newSize, std::numeric_limits<int>::max());
516 domesticOverlapByIndex_.resize(newSize);
523 domesticOverlapByIndex_[
static_cast<unsigned>(domesticIdx)][static_cast<unsigned>(peerRank)] = borderDistance;
524 domesticOverlapWithPeer_[
static_cast<unsigned>(peerRank)].push_back(domesticIdx);
527 assert(globalIdx >= 0);
528 assert(domesticIdx >= 0);
529 assert(!(borderDistance == 0 && !foreignOverlap_.
isLocal(domesticIdx)));
530 assert(!(borderDistance > 0 && foreignOverlap_.
isLocal(domesticIdx)));
532 borderDistance_[
static_cast<unsigned>(domesticIdx)] = std::min(borderDistance, borderDistance_[static_cast<unsigned>(domesticIdx)]);
546 void setupDebugMapping_()
553 Index mapInternalToExternal_(Index internalIdx)
const
554 {
return internalIdx; }
560 Index mapExternalToInternal_(Index externalIdx)
const
561 {
return externalIdx; }
565 ForeignOverlap foreignOverlap_;
567 BlackList blackList_;
569 DomesticOverlapByRank domesticOverlapWithPeer_;
570 OverlapByIndex domesticOverlapByIndex_;
571 std::vector<BorderDistance> borderDistance_;
572 std::vector<ProcessRank> masterRank_;
574 std::map<ProcessRank, MpiBuffer<size_t> *> numIndicesSendBuffer_;
575 std::map<ProcessRank, MpiBuffer<IndexDistanceNpeers> *> indicesSendBuffer_;
576 GlobalIndices globalIndices_;
Simplifies handling of buffers to be used in conjunction with MPI.
Definition: mpibuffer.hh:45
bool iAmMasterOf(Index localIdx) const
Return true if the current rank is the "master" of an index.
Definition: foreignoverlapfrombcrsmatrix.hh:184
bool isFront(Index domesticIdx) const
Returns true iff a domestic index is on the front.
Definition: domesticoverlapfrombcrsmatrix.hh:178
Index domesticToGlobal(Index domIdx) const
Returns a global index given a domestic one.
Definition: domesticoverlapfrombcrsmatrix.hh:285
bool peerHasIndex(int peerRank, Index domesticIdx) const
Returns true iff a domestic index is seen by a peer rank.
Definition: domesticoverlapfrombcrsmatrix.hh:333
Index foreignOverlapOffsetToDomesticIdx(ProcessRank peerRank, unsigned overlapOffset) const
Returns the domestic index given an offset in the foreign overlap of a peer process with the local pr...
Definition: domesticoverlapfrombcrsmatrix.hh:351
bool isBorderWith(Index localIdx, ProcessRank peerRank) const
Returns true iff a local index is a border index shared with a given peer process.
Definition: foreignoverlapfrombcrsmatrix.hh:158
size_t numDomestic() const
Returns the number domestic indices.
Definition: globalindices.hh:121
unsigned overlapSize() const
Returns the size of the overlap region.
Definition: domesticoverlapfrombcrsmatrix.hh:207
bool isInOverlap(Index localIdx) const
Returns true if a given local index is in the foreign overlap of any rank.
Definition: foreignoverlapfrombcrsmatrix.hh:325
size_t numFront(ProcessRank peerRank) const
Returns the number of front indices of a peer process in the local partition.
Definition: foreignoverlapfrombcrsmatrix.hh:229
Index domesticOverlapOffsetToDomesticIdx(ProcessRank peerRank, Index overlapOffset) const
Returns the domestic index given an offset in the domestic overlap of a peer process with the local p...
Definition: domesticoverlapfrombcrsmatrix.hh:370
const PeerSet & peerSet() const
Return the set of process ranks which share an overlap with the current process.
Definition: foreignoverlapfrombcrsmatrix.hh:262
Expresses which degrees of freedom are blacklisted for the parallel linear solvers and which domestic...
bool isBorderWith(Index domesticIdx, ProcessRank peerRank) const
Returns true iff a domestic index is on the border with a given peer process.
Definition: domesticoverlapfrombcrsmatrix.hh:161
void print() const
Print the foreign overlap for debugging purposes.
Definition: domesticoverlapfrombcrsmatrix.hh:268
bool isFrontFor(ProcessRank peerRank, Index localIdx) const
Returns whether a given local index is on the front of a given peer rank.
Definition: foreignoverlapfrombcrsmatrix.hh:247
Simplifies handling of buffers to be used in conjunction with MPI.
size_t numLocal() const
Returns the number local indices.
Definition: domesticoverlapfrombcrsmatrix.hh:226
DomesticOverlapFromBCRSMatrix(const BCRSMatrix &A, const BorderList &borderList, const BlackList &blackList, unsigned overlapSize)
Constructs the foreign overlap given a BCRS matrix and an initial list of border indices.
Definition: domesticoverlapfrombcrsmatrix.hh:66
Index globalToDomestic(Index globalIdx) const
Returns a domestic index given a global one.
Definition: domesticoverlapfrombcrsmatrix.hh:274
const BlackList & blackList() const
Returns the object which represents the black-listed native indices.
Definition: domesticoverlapfrombcrsmatrix.hh:194
size_t numNative() const
Returns the number native indices.
Definition: domesticoverlapfrombcrsmatrix.hh:217
bool isInOverlap(Index domesticIdx) const
Returns true if a given domestic index is either in the foreign or in the domestic overlap...
Definition: domesticoverlapfrombcrsmatrix.hh:314
This class maps domestic row indices to and from "global" indices which is used to construct an algeb...
Index nativeToDomestic(Index nativeIdx) const
Returns a domestic index given a native one.
Definition: domesticoverlapfrombcrsmatrix.hh:302
const std::map< ProcessRank, BorderDistance > & foreignOverlapByLocalIndex(Index localIdx) const
Return the map of (peer rank, border distance) for a given local index.
Definition: foreignoverlapfrombcrsmatrix.hh:210
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
Definition: foreignoverlapfrombcrsmatrix.hh:61
This class maps domestic row indices to and from "global" indices which is used to construct an algeb...
Definition: globalindices.hh:55
bool iAmMasterOf(Index domesticIdx) const
Return true iff the current process is the master of a given domestic index.
Definition: domesticoverlapfrombcrsmatrix.hh:252
size_t foreignOverlapSize(ProcessRank peerRank) const
Returns number of indices which are contained in the foreign overlap with a peer. ...
Definition: domesticoverlapfrombcrsmatrix.hh:343
unsigned overlapSize() const
Returns the size of the overlap region.
Definition: foreignoverlapfrombcrsmatrix.hh:145
void addIndex(Index domesticIdx, Index globalIdx)
Add an index to the domestic<->global mapping.
Definition: globalindices.hh:127
bool hasGlobalIndex(Index globalIdx) const
Return true iff a given global index already exists.
Definition: globalindices.hh:181
Expresses which degrees of freedom are blacklisted for the parallel linear solvers and which domestic...
Definition: blacklist.hh:47
bool peerHasIndex(ProcessRank peerRank, Index localIdx) const
Returns true iff a local index is seen by a peer rank.
Definition: foreignoverlapfrombcrsmatrix.hh:219
size_t numLocal() const
Returns the number of local indices.
Definition: foreignoverlapfrombcrsmatrix.hh:281
bool isBorder(Index domesticIdx) const
Returns true iff a domestic index is a border index.
Definition: domesticoverlapfrombcrsmatrix.hh:151
unsigned worldSize() const
Returns the number of processes in the global MPI communicator.
Definition: domesticoverlapfrombcrsmatrix.hh:138
const PeerSet & peerSet() const
Return the set of process ranks which share an overlap with the current process.
Definition: domesticoverlapfrombcrsmatrix.hh:145
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
size_t numDomestic() const
Returns the number domestic indices.
Definition: domesticoverlapfrombcrsmatrix.hh:236
Index domesticToGlobal(Index domesticIdx) const
Converts a domestic index to a global one.
Definition: globalindices.hh:88
ProcessRank myRank() const
Returns the rank of the current process.
Definition: domesticoverlapfrombcrsmatrix.hh:132
size_t numNative() const
Returns the number of native indices.
Definition: foreignoverlapfrombcrsmatrix.hh:275
Index nativeToLocal(Index nativeIdx) const
Convert a native index to a local one.
Definition: foreignoverlapfrombcrsmatrix.hh:296
ProcessRank masterRank(Index localIdx) const
Return the rank of the master process of an index.
Definition: foreignoverlapfrombcrsmatrix.hh:173
Index globalToDomestic(Index globalIdx) const
Converts a global index to a domestic one.
Definition: globalindices.hh:98
size_t numFront(ProcessRank peerRank) const
Returns the number of indices on the front within a given peer rank's grid partition.
Definition: domesticoverlapfrombcrsmatrix.hh:172
Index localToNative(Index localIdx) const
Convert a local index to a native one.
Definition: foreignoverlapfrombcrsmatrix.hh:302
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
Definition: domesticoverlapfrombcrsmatrix.hh:52
const OverlapWithPeer & foreignOverlapWithPeer(ProcessRank peerRank) const
Return the list of (local indices, border distance, number of processes) triples which are in the ove...
Definition: foreignoverlapfrombcrsmatrix.hh:199
Index domesticToNative(Index domIdx) const
Returns a native index given a domestic one.
Definition: domesticoverlapfrombcrsmatrix.hh:291
bool isLocal(Index domesticIdx) const
Return true if a domestic index is local for the process.
Definition: domesticoverlapfrombcrsmatrix.hh:245
size_t domesticOverlapSize(ProcessRank peerRank) const
Returns number of indices which are contained in the domestic overlap with a peer.
Definition: domesticoverlapfrombcrsmatrix.hh:362
size_t numPeers(Index domesticIdx) const
Returns the number of processes which "see" a given index.
Definition: domesticoverlapfrombcrsmatrix.hh:201
bool isLocal(Index domesticIdx) const
Returns true iff a domestic index is local.
Definition: foreignoverlapfrombcrsmatrix.hh:287
A set of process ranks.
Definition: overlaptypes.hh:148
bool isBorder(Index localIdx) const
Returns true iff a local index is a border index.
Definition: foreignoverlapfrombcrsmatrix.hh:151
bool isFrontFor(ProcessRank peerRank, Index domesticIdx) const
Returns true if a given domestic index is a front index for a peer rank.
Definition: domesticoverlapfrombcrsmatrix.hh:324
ProcessRank masterRank(Index domesticIdx) const
Return the rank of a master process for a domestic index.
Definition: domesticoverlapfrombcrsmatrix.hh:262
void print() const
Prints the global indices of all domestic indices for debugging purposes.
Definition: globalindices.hh:188