00001
00002
00003 #ifndef DUNE_INDICESSYNCER_HH
00004 #define DUNE_INDICESSYNCER_HH
00005
00006 #include "indexset.hh"
00007 #include "remoteindices.hh"
00008 #include <dune/common/stdstreams.hh>
00009 #include <dune/common/sllist.hh>
00010 #include <dune/common/unused.hh>
00011 #include <cassert>
00012 #include <cmath>
00013 #include <limits>
00014 #include <algorithm>
00015 #include <functional>
00016 #include <map>
00017 #include <tuple>
00018
00019 #if HAVE_MPI
00020 namespace Dune
00021 {
00038 template<typename T>
00039 class IndicesSyncer
00040 {
00041 public:
00042
00044 typedef T ParallelIndexSet;
00045
00047 typedef typename ParallelIndexSet::IndexPair IndexPair;
00048
00050 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00051
00053 typedef typename ParallelIndexSet::LocalIndex::Attribute Attribute;
00054
00058 typedef Dune::RemoteIndices<ParallelIndexSet> RemoteIndices;
00059
00069 IndicesSyncer(ParallelIndexSet& indexSet,
00070 RemoteIndices& remoteIndices);
00071
00081 void sync();
00082
00093 template<typename T1>
00094 void sync(T1& numberer);
00095
00096 private:
00097
00099 ParallelIndexSet& indexSet_;
00100
00102 RemoteIndices& remoteIndices_;
00103
00105 char** sendBuffers_;
00106
00108 char* receiveBuffer_;
00109
00111 std::size_t* sendBufferSizes_;
00112
00114 int receiveBufferSize_;
00115
00119 struct MessageInformation
00120 {
00121 MessageInformation()
00122 : publish(), pairs()
00123 {}
00125 int publish;
00130 int pairs;
00131 };
00132
00136 class DefaultNumberer
00137 {
00138 public:
00144 std::size_t operator()(const GlobalIndex& global)
00145 {
00146 DUNE_UNUSED_PARAMETER(global);
00147 return std::numeric_limits<size_t>::max();
00148 }
00149 };
00150
00152 MPI_Datatype datatype_;
00153
00155 int rank_;
00156
00161 typedef SLList<std::pair<GlobalIndex,Attribute>, typename RemoteIndices::Allocator> GlobalIndexList;
00162
00164 typedef typename GlobalIndexList::ModifyIterator GlobalIndexModifier;
00165
00169 typedef typename SLList<GlobalIndex, typename RemoteIndices::Allocator>::iterator
00170 GlobalIndexIterator;
00171
00173 typedef std::map<int, GlobalIndexList> GlobalIndicesMap;
00174
00183 GlobalIndicesMap globalMap_;
00184
00188 typedef SLList<bool, typename RemoteIndices::Allocator> BoolList;
00189
00193 typedef typename BoolList::iterator BoolIterator;
00194
00196 typedef typename BoolList::ModifyIterator BoolListModifier;
00197
00199 typedef std::map<int,BoolList> BoolMap;
00200
00205 BoolMap oldMap_;
00206
00208 std::map<int,MessageInformation> infoSend_;
00209
00211 typedef typename RemoteIndices::RemoteIndexList RemoteIndexList;
00212
00214 typedef typename RemoteIndexList::ModifyIterator RemoteIndexModifier;
00215
00217 typedef Dune::RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00218
00220 typedef typename RemoteIndexList::iterator RemoteIndexIterator;
00221
00223 typedef typename RemoteIndexList::const_iterator ConstRemoteIndexIterator;
00224
00226 typedef std::tuple<RemoteIndexModifier,GlobalIndexModifier,BoolListModifier,
00227 const ConstRemoteIndexIterator> IteratorTuple;
00228
00236 class Iterators
00237 {
00238 friend class IndicesSyncer<T>;
00239 public:
00249 Iterators(RemoteIndexList& remoteIndices, GlobalIndexList& globalIndices,
00250 BoolList& booleans);
00251
00255 Iterators();
00256
00260 Iterators& operator++();
00261
00267 void insert(const RemoteIndex& index,
00268 const std::pair<GlobalIndex,Attribute>& global);
00269
00274 RemoteIndex& remoteIndex() const;
00275
00280 std::pair<GlobalIndex,Attribute>& globalIndexPair() const;
00281
00282 Attribute& attribute() const;
00283
00289 bool isOld() const;
00290
00300 void reset(RemoteIndexList& remoteIndices, GlobalIndexList& globalIndices,
00301 BoolList& booleans);
00302
00308 bool isNotAtEnd() const;
00309
00315 bool isAtEnd() const;
00316
00317 private:
00327 IteratorTuple iterators_;
00328 };
00329
00331 typedef std::map<int,Iterators> IteratorsMap;
00332
00344 IteratorsMap iteratorsMap_;
00345
00347 void calculateMessageSizes();
00348
00356 void packAndSend(int destination, char* buffer, std::size_t bufferSize, MPI_Request& req);
00357
00362 template<typename T1>
00363 void recvAndUnpack(T1& numberer);
00364
00368 void registerMessageDatatype();
00369
00373 void insertIntoRemoteIndexList(int process,
00374 const std::pair<GlobalIndex,Attribute>& global,
00375 char attribute);
00376
00380 void resetIteratorsMap();
00381
00386 bool checkReset();
00387
00396 bool checkReset(const Iterators& iterators, RemoteIndexList& rlist, GlobalIndexList& gList,
00397 BoolList& bList);
00398 };
00399
00400 template<typename TG, typename TA>
00401 bool operator<(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
00402 const std::pair<TG,TA>& i2)
00403 {
00404 return i1.global() < i2.first ||
00405 (i1.global() == i2.first && i1.local().attribute()<i2.second);
00406 }
00407
00408 template<typename TG, typename TA>
00409 bool operator<(const std::pair<TG,TA>& i1,
00410 const IndexPair<TG,ParallelLocalIndex<TA> >& i2)
00411 {
00412 return i1.first < i2.global() ||
00413 (i1.first == i2.global() && i1.second<i2.local().attribute());
00414 }
00415
00416 template<typename TG, typename TA>
00417 bool operator==(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
00418 const std::pair<TG,TA>& i2)
00419 {
00420 return (i1.global() == i2.first && i1.local().attribute()==i2.second);
00421 }
00422
00423 template<typename TG, typename TA>
00424 bool operator!=(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
00425 const std::pair<TG,TA>& i2)
00426 {
00427 return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
00428 }
00429
00430 template<typename TG, typename TA>
00431 bool operator==(const std::pair<TG,TA>& i2,
00432 const IndexPair<TG,ParallelLocalIndex<TA> >& i1)
00433 {
00434 return (i1.global() == i2.first && i1.local().attribute()==i2.second);
00435 }
00436
00437 template<typename TG, typename TA>
00438 bool operator!=(const std::pair<TG,TA>& i2,
00439 const IndexPair<TG,ParallelLocalIndex<TA> >& i1)
00440 {
00441 return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
00442 }
00443
00460 template<typename T, typename A, typename A1>
00461 void storeGlobalIndicesOfRemoteIndices(std::map<int,SLList<std::pair<typename T::GlobalIndex, typename T::LocalIndex::Attribute>,A> >& globalMap,
00462 const RemoteIndices<T,A1>& remoteIndices)
00463 {
00464 typedef typename RemoteIndices<T,A1>::const_iterator RemoteIterator;
00465
00466 for(RemoteIterator remote = remoteIndices.begin(), end =remoteIndices.end(); remote != end; ++remote) {
00467 typedef typename RemoteIndices<T,A1>::RemoteIndexList RemoteIndexList;
00468 typedef typename RemoteIndexList::const_iterator RemoteIndexIterator;
00469 typedef SLList<std::pair<typename T::GlobalIndex, typename T::LocalIndex::Attribute>,A> GlobalIndexList;
00470 GlobalIndexList& global = globalMap[remote->first];
00471 RemoteIndexList& rList = *(remote->second.first);
00472
00473 for(RemoteIndexIterator index = rList.begin(), riEnd = rList.end();
00474 index != riEnd; ++index) {
00475 global.push_back(std::make_pair(index->localIndexPair().global(),
00476 index->localIndexPair().local().attribute()));
00477 }
00478 }
00479 }
00480
00489 template<typename T, typename A, typename A1>
00490 inline void repairLocalIndexPointers(std::map<int,
00491 SLList<std::pair<typename T::GlobalIndex,
00492 typename T::LocalIndex::Attribute>,A> >& globalMap,
00493 RemoteIndices<T,A1>& remoteIndices,
00494 const T& indexSet)
00495 {
00496 typedef typename RemoteIndices<T,A1>::RemoteIndexMap::iterator RemoteIterator;
00497 typedef typename RemoteIndices<T,A1>::RemoteIndexList::iterator RemoteIndexIterator;
00498 typedef typename T::GlobalIndex GlobalIndex;
00499 typedef typename T::LocalIndex::Attribute Attribute;
00500 typedef std::pair<GlobalIndex,Attribute> GlobalIndexPair;
00501 typedef SLList<GlobalIndexPair,A> GlobalIndexPairList;
00502 typedef typename GlobalIndexPairList::iterator GlobalIndexIterator;
00503
00504 assert(globalMap.size()==static_cast<std::size_t>(remoteIndices.neighbours()));
00505
00506 typename std::map<int,GlobalIndexPairList>::iterator global = globalMap.begin();
00507 RemoteIterator end = remoteIndices.remoteIndices_.end();
00508
00509 for(RemoteIterator remote = remoteIndices.remoteIndices_.begin(); remote != end; ++remote, ++global) {
00510 typedef typename T::const_iterator IndexIterator;
00511
00512 assert(remote->first==global->first);
00513 assert(remote->second.first->size() == global->second.size());
00514
00515 RemoteIndexIterator riEnd = remote->second.first->end();
00516 RemoteIndexIterator rIndex = remote->second.first->begin();
00517 GlobalIndexIterator gIndex = global->second.begin();
00518 IndexIterator index = indexSet.begin();
00519
00520 assert(rIndex==riEnd || gIndex != global->second.end());
00521 while(rIndex != riEnd) {
00522
00523 assert(gIndex != global->second.end());
00524
00525 while(!(index->global() == gIndex->first
00526 && index->local().attribute() == gIndex->second)) {
00527 ++index;
00528
00529
00530
00531 if (index->global() > gIndex->first) {
00532 index=indexSet.begin();
00533 }
00534 }
00535
00536 assert(index != indexSet.end() && *index == *gIndex);
00537
00538 rIndex->localIndex_ = &(*index);
00539 ++index;
00540 ++rIndex;
00541 ++gIndex;
00542 }
00543 }
00544 remoteIndices.sourceSeqNo_ = remoteIndices.source_->seqNo();
00545 remoteIndices.destSeqNo_ = remoteIndices.target_->seqNo();
00546 }
00547
00548 template<typename T>
00549 IndicesSyncer<T>::IndicesSyncer(ParallelIndexSet& indexSet,
00550 RemoteIndices& remoteIndices)
00551 : indexSet_(indexSet), remoteIndices_(remoteIndices)
00552 {
00553
00554 assert(remoteIndices.source_ == remoteIndices.target_);
00555 assert(remoteIndices.source_ == &indexSet);
00556 MPI_Comm_rank(remoteIndices_.communicator(), &rank_);
00557 }
00558
00559 template<typename T>
00560 IndicesSyncer<T>::Iterators::Iterators(RemoteIndexList& remoteIndices,
00561 GlobalIndexList& globalIndices,
00562 BoolList& booleans)
00563 : iterators_(remoteIndices.beginModify(), globalIndices.beginModify(),
00564 booleans.beginModify(), remoteIndices.end())
00565 { }
00566
00567 template<typename T>
00568 IndicesSyncer<T>::Iterators::Iterators()
00569 : iterators_()
00570 {}
00571
00572 template<typename T>
00573 inline typename IndicesSyncer<T>::Iterators& IndicesSyncer<T>::Iterators::operator++()
00574 {
00575 ++(std::get<0>(iterators_));
00576 ++(std::get<1>(iterators_));
00577 ++(std::get<2>(iterators_));
00578 return *this;
00579 }
00580
00581 template<typename T>
00582 inline void IndicesSyncer<T>::Iterators::insert(const RemoteIndex & index,
00583 const std::pair<GlobalIndex,Attribute>& global)
00584 {
00585 std::get<0>(iterators_).insert(index);
00586 std::get<1>(iterators_).insert(global);
00587 std::get<2>(iterators_).insert(false);
00588 }
00589
00590 template<typename T>
00591 inline typename IndicesSyncer<T>::RemoteIndex&
00592 IndicesSyncer<T>::Iterators::remoteIndex() const
00593 {
00594 return *(std::get<0>(iterators_));
00595 }
00596
00597 template<typename T>
00598 inline std::pair<typename IndicesSyncer<T>::GlobalIndex,typename IndicesSyncer<T>::Attribute>&
00599 IndicesSyncer<T>::Iterators::globalIndexPair() const
00600 {
00601 return *(std::get<1>(iterators_));
00602 }
00603
00604 template<typename T>
00605 inline bool IndicesSyncer<T>::Iterators::isOld() const
00606 {
00607 return *(std::get<2>(iterators_));
00608 }
00609
00610 template<typename T>
00611 inline void IndicesSyncer<T>::Iterators::reset(RemoteIndexList& remoteIndices,
00612 GlobalIndexList& globalIndices,
00613 BoolList& booleans)
00614 {
00615 std::get<0>(iterators_) = remoteIndices.beginModify();
00616 std::get<1>(iterators_) = globalIndices.beginModify();
00617 std::get<2>(iterators_) = booleans.beginModify();
00618 }
00619
00620 template<typename T>
00621 inline bool IndicesSyncer<T>::Iterators::isNotAtEnd() const
00622 {
00623 return std::get<0>(iterators_) != std::get<3>(iterators_);
00624 }
00625
00626 template<typename T>
00627 inline bool IndicesSyncer<T>::Iterators::isAtEnd() const
00628 {
00629 return std::get<0>(iterators_) == std::get<3>(iterators_);
00630 }
00631
00632 template<typename T>
00633 void IndicesSyncer<T>::registerMessageDatatype()
00634 {
00635 MPI_Datatype type[2] = {MPI_INT, MPI_INT};
00636 int blocklength[2] = {1,1};
00637 MPI_Aint displacement[2];
00638 MPI_Aint base;
00639
00640
00641 MessageInformation message;
00642
00643 MPI_Get_address( &(message.publish), displacement);
00644 MPI_Get_address( &(message.pairs), displacement+1);
00645
00646
00647 MPI_Get_address(&message, &base);
00648 displacement[0] -= base;
00649 displacement[1] -= base;
00650
00651 MPI_Type_create_struct( 2, blocklength, displacement, type, &datatype_);
00652 MPI_Type_commit(&datatype_);
00653 }
00654
00655 template<typename T>
00656 void IndicesSyncer<T>::calculateMessageSizes()
00657 {
00658 typedef typename ParallelIndexSet::const_iterator IndexIterator;
00659 typedef CollectiveIterator<T,typename RemoteIndices::Allocator> CollectiveIterator;
00660
00661 IndexIterator iEnd = indexSet_.end();
00662 CollectiveIterator collIter = remoteIndices_.template iterator<true>();
00663
00664 for(IndexIterator index = indexSet_.begin(); index != iEnd; ++index) {
00665 collIter.advance(index->global(), index->local().attribute());
00666 if(collIter.empty())
00667 break;
00668 int knownRemote=0;
00669
00670 typedef typename CollectiveIterator::iterator ValidIterator;
00671 ValidIterator end = collIter.end();
00672
00673
00674 for(ValidIterator valid = collIter.begin(); valid != end; ++valid) {
00675 ++knownRemote;
00676 }
00677
00678 if(knownRemote>0) {
00679 Dune::dverb<<rank_<<": publishing "<<knownRemote<<" for index "<<index->global()<< " for processes ";
00680
00681
00682 for(ValidIterator valid = collIter.begin(); valid != end; ++valid) {
00683 ++(infoSend_[valid.process()].publish);
00684 (infoSend_[valid.process()].pairs) += knownRemote;
00685 Dune::dverb<<valid.process()<<" ";
00686 Dune::dverb<<"(publish="<<infoSend_[valid.process()].publish<<", pairs="<<infoSend_[valid.process()].pairs
00687 <<") ";
00688 }
00689 Dune::dverb<<std::endl;
00690 }
00691 }
00692
00693 typedef typename std::map<int,MessageInformation>::const_iterator
00694 MessageIterator;
00695
00696 const MessageIterator end = infoSend_.end();
00697
00698
00699
00700
00701 MessageInformation dummy;
00702
00703 MessageIterator messageIter= infoSend_.begin();
00704
00705 typedef typename RemoteIndices::RemoteIndexMap::const_iterator RemoteIterator;
00706 const RemoteIterator rend = remoteIndices_.end();
00707 int neighbour=0;
00708
00709 for(RemoteIterator remote = remoteIndices_.begin(); remote != rend; ++remote, ++neighbour) {
00710 MessageInformation* message;
00711 MessageInformation recv;
00712
00713 if(messageIter != end && messageIter->first==remote->first) {
00714
00715 message = const_cast<MessageInformation*>(&(messageIter->second));
00716 ++messageIter;
00717 }else
00718
00719 message = &dummy;
00720
00721 sendBufferSizes_[neighbour]=0;
00722 int tsize;
00723
00724 MPI_Pack_size(1, MPI_INT,remoteIndices_.communicator(), &tsize);
00725 sendBufferSizes_[neighbour] += tsize;
00726
00727 for(int i=0; i < message->publish; ++i) {
00728
00729 MPI_Pack_size(1, MPITraits<GlobalIndex>::getType(), remoteIndices_.communicator(), &tsize);
00730 sendBufferSizes_[neighbour] += tsize;
00731
00732 MPI_Pack_size(1, MPI_CHAR, remoteIndices_.communicator(), &tsize);
00733 sendBufferSizes_[neighbour] += tsize;
00734
00735 MPI_Pack_size(1, MPI_INT, remoteIndices_.communicator(), &tsize);
00736 sendBufferSizes_[neighbour] += tsize;
00737 }
00738 for(int i=0; i < message->pairs; ++i) {
00739
00740 MPI_Pack_size(1, MPI_INT, remoteIndices_.communicator(), &tsize);
00741 sendBufferSizes_[neighbour] += tsize;
00742
00743 MPI_Pack_size(1, MPI_CHAR, remoteIndices_.communicator(), &tsize);
00744 sendBufferSizes_[neighbour] += tsize;
00745 }
00746
00747 Dune::dverb<<rank_<<": Buffer (neighbour="<<remote->first<<") size is "<< sendBufferSizes_[neighbour]<<" for publish="<<message->publish<<" pairs="<<message->pairs<<std::endl;
00748 }
00749
00750 }
00751
00752 template<typename T>
00753 inline void IndicesSyncer<T>::sync()
00754 {
00755 DefaultNumberer numberer;
00756 sync(numberer);
00757 }
00758
00759 template<typename T>
00760 template<typename T1>
00761 void IndicesSyncer<T>::sync(T1& numberer)
00762 {
00763
00764
00765
00766
00767
00768
00769 typedef typename RemoteIndices::RemoteIndexMap::const_iterator
00770 RemoteIterator;
00771
00772 const RemoteIterator end = remoteIndices_.end();
00773
00774
00775
00776 std::size_t noOldNeighbours = remoteIndices_.neighbours();
00777 int* oldNeighbours = new int[noOldNeighbours];
00778 sendBufferSizes_ = new std::size_t[noOldNeighbours];
00779 std::size_t neighbourI = 0;
00780
00781 for(RemoteIterator remote = remoteIndices_.begin(); remote != end; ++remote, ++neighbourI) {
00782 typedef typename RemoteIndices::RemoteIndexList::const_iterator
00783 RemoteIndexIterator;
00784
00785 oldNeighbours[neighbourI] = remote->first;
00786
00787
00788 assert(remote->second.first==remote->second.second);
00789
00790 RemoteIndexList& rList = *(remote->second.first);
00791
00792
00793 GlobalIndexList& global = globalMap_[remote->first];
00794 BoolList& added = oldMap_[remote->first];
00795 RemoteIndexIterator riEnd = rList.end();
00796
00797 for(RemoteIndexIterator index = rList.begin();
00798 index != riEnd; ++index) {
00799 global.push_back(std::make_pair(index->localIndexPair().global(),
00800 index->localIndexPair().local().attribute()));
00801 added.push_back(true);
00802 }
00803
00804 Iterators iterators(rList, global, added);
00805 iteratorsMap_.insert(std::make_pair(remote->first, iterators));
00806 assert(checkReset(iteratorsMap_[remote->first], rList,global,added));
00807 }
00808
00809
00810 calculateMessageSizes();
00811
00812
00813 receiveBufferSize_=1;
00814 sendBuffers_ = new char*[noOldNeighbours];
00815
00816 for(std::size_t i=0; i<noOldNeighbours; ++i) {
00817 sendBuffers_[i] = new char[sendBufferSizes_[i]];
00818 receiveBufferSize_ = std::max(receiveBufferSize_, static_cast<int>(sendBufferSizes_[i]));
00819 }
00820
00821 receiveBuffer_=new char[receiveBufferSize_];
00822
00823 indexSet_.beginResize();
00824
00825 Dune::dverb<<rank_<<": Neighbours: ";
00826
00827 for(std::size_t i = 0; i<noOldNeighbours; ++i)
00828 Dune::dverb<<oldNeighbours[i]<<" ";
00829
00830 Dune::dverb<<std::endl;
00831
00832 MPI_Request* requests = new MPI_Request[noOldNeighbours];
00833 MPI_Status* statuses = new MPI_Status[noOldNeighbours];
00834
00835
00836 for(std::size_t i = 0; i<noOldNeighbours; ++i)
00837 packAndSend(oldNeighbours[i], sendBuffers_[i], sendBufferSizes_[i], requests[i]);
00838
00839
00840 for(std::size_t i = 0; i<noOldNeighbours; ++i)
00841 recvAndUnpack(numberer);
00842
00843
00844
00845
00846
00847
00848 delete[] receiveBuffer_;
00849
00850
00851
00852 if(MPI_SUCCESS!=MPI_Waitall(noOldNeighbours, requests, statuses)) {
00853 std::cerr<<": MPI_Error occurred while sending message"<<std::endl;
00854 for(std::size_t i=0; i< noOldNeighbours; i++)
00855 if(MPI_SUCCESS!=statuses[i].MPI_ERROR)
00856 std::cerr<<"Destination "<<statuses[i].MPI_SOURCE<<" error code: "<<statuses[i].MPI_ERROR<<std::endl;
00857 }
00858
00859 delete[] statuses;
00860 delete[] requests;
00861
00862 for(std::size_t i=0; i<noOldNeighbours; ++i)
00863 delete[] sendBuffers_[i];
00864
00865 delete[] sendBuffers_;
00866 delete[] sendBufferSizes_;
00867
00868
00869 iteratorsMap_.clear();
00870
00871 indexSet_.endResize();
00872
00873 delete[] oldNeighbours;
00874
00875 repairLocalIndexPointers(globalMap_, remoteIndices_, indexSet_);
00876
00877 oldMap_.clear();
00878 globalMap_.clear();
00879
00880
00881 remoteIndices_.sourceSeqNo_ = remoteIndices_.destSeqNo_ = indexSet_.seqNo();
00882 }
00883
00884 template<typename T>
00885 void IndicesSyncer<T>::packAndSend(int destination, char* buffer, std::size_t bufferSize, MPI_Request& request)
00886 {
00887 typedef typename ParallelIndexSet::const_iterator IndexIterator;
00888
00889 IndexIterator iEnd = indexSet_.end();
00890 int bpos = 0;
00891 int published = 0;
00892 int pairs = 0;
00893
00894 assert(checkReset());
00895
00896
00897 MPI_Pack(&(infoSend_[destination].publish), 1, MPI_INT, buffer, bufferSize, &bpos,
00898 remoteIndices_.communicator());
00899
00900 for(IndexIterator index = indexSet_.begin(); index != iEnd; ++index) {
00901
00902 typedef typename IteratorsMap::iterator Iterator;
00903 Iterator iteratorsEnd = iteratorsMap_.end();
00904
00905
00906 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators) {
00907 while(iterators->second.isNotAtEnd() &&
00908 iterators->second.globalIndexPair().first < index->global())
00909 ++(iterators->second);
00910 assert(!iterators->second.isNotAtEnd() || iterators->second.globalIndexPair().first >= index->global());
00911 }
00912
00913
00914
00915
00916 int indices = 0;
00917 bool knownRemote = false;
00918
00919 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
00920 {
00921 std::pair<GlobalIndex,Attribute> p;
00922 if (iterators->second.isNotAtEnd())
00923 {
00924 p = iterators->second.globalIndexPair();
00925 }
00926
00927 if(iterators->second.isNotAtEnd() && iterators->second.isOld()
00928 && iterators->second.globalIndexPair().first == index->global()) {
00929 indices++;
00930 if(destination == iterators->first)
00931 knownRemote = true;
00932 }
00933 }
00934
00935 if(!knownRemote)
00936
00937 continue;
00938
00939 Dune::dverb<<rank_<<": sending "<<indices<<" for index "<<index->global()<<" to "<<destination<<std::endl;
00940
00941
00942
00943 MPI_Pack(const_cast<GlobalIndex*>(&(index->global())), 1, MPITraits<GlobalIndex>::getType(), buffer, bufferSize, &bpos,
00944 remoteIndices_.communicator());
00945
00946 char attr = index->local().attribute();
00947 MPI_Pack(&attr, 1, MPI_CHAR, buffer, bufferSize, &bpos,
00948 remoteIndices_.communicator());
00949
00950
00951 MPI_Pack(&indices, 1, MPI_INT, buffer, bufferSize, &bpos,
00952 remoteIndices_.communicator());
00953
00954
00955 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
00956 if(iterators->second.isNotAtEnd() && iterators->second.isOld()
00957 && iterators->second.globalIndexPair().first == index->global()) {
00958 int process = iterators->first;
00959
00960 ++pairs;
00961 assert(pairs <= infoSend_[destination].pairs);
00962 MPI_Pack(&process, 1, MPI_INT, buffer, bufferSize, &bpos,
00963 remoteIndices_.communicator());
00964 char attr2 = iterators->second.remoteIndex().attribute();
00965
00966 MPI_Pack(&attr2, 1, MPI_CHAR, buffer, bufferSize, &bpos,
00967 remoteIndices_.communicator());
00968 --indices;
00969 }
00970 assert(indices==0);
00971 ++published;
00972 Dune::dvverb<<" (publish="<<published<<", pairs="<<pairs<<")"<<std::endl;
00973 assert(published <= infoSend_[destination].publish);
00974 }
00975
00976
00977 assert(published == infoSend_[destination].publish);
00978 assert(pairs == infoSend_[destination].pairs);
00979 resetIteratorsMap();
00980
00981 Dune::dverb << rank_<<": Sending message of "<<bpos<<" bytes to "<<destination<<std::endl;
00982
00983 MPI_Issend(buffer, bpos, MPI_PACKED, destination, 345, remoteIndices_.communicator(),&request);
00984 }
00985
00986 template<typename T>
00987 inline void IndicesSyncer<T>::insertIntoRemoteIndexList(int process,
00988 const std::pair<GlobalIndex,Attribute>& globalPair,
00989 char attribute)
00990 {
00991 Dune::dverb<<"Inserting from "<<process<<" "<<globalPair.first<<", "<<
00992 globalPair.second<<" "<<attribute<<std::endl;
00993
00994 resetIteratorsMap();
00995
00996
00997 typename IteratorsMap::iterator found = iteratorsMap_.find(process);
00998
00999 if( found == iteratorsMap_.end() ) {
01000 Dune::dverb<<"Discovered new neighbour "<<process<<std::endl;
01001 RemoteIndexList* rlist = new RemoteIndexList();
01002 remoteIndices_.remoteIndices_.insert(std::make_pair(process,std::make_pair(rlist,rlist)));
01003 Iterators iterators = Iterators(*rlist, globalMap_[process], oldMap_[process]);
01004 found = iteratorsMap_.insert(std::make_pair(process, iterators)).first;
01005 }
01006
01007 Iterators& iterators = found->second;
01008
01009
01010 while(iterators.isNotAtEnd() && iterators.globalIndexPair() < globalPair) {
01011
01012 ++iterators;
01013
01014 }
01015
01016 if(iterators.isAtEnd() || iterators.globalIndexPair() != globalPair) {
01017
01018
01019 iterators.insert(RemoteIndex(Attribute(attribute)),globalPair);
01020 return;
01021 }
01022
01023
01024 bool indexIsThere=false;
01025 for(Iterators tmpIterators = iterators;
01026 !tmpIterators.isAtEnd() && tmpIterators.globalIndexPair() == globalPair;
01027 ++tmpIterators)
01028
01029 if(tmpIterators.globalIndexPair().second == attribute) {
01030 indexIsThere=true;
01031 break;
01032 }
01033
01034 if(!indexIsThere)
01035
01036
01037 iterators.insert(RemoteIndex(Attribute(attribute)),globalPair);
01038 }
01039
01040 template<typename T>
01041 template<typename T1>
01042 void IndicesSyncer<T>::recvAndUnpack(T1& numberer)
01043 {
01044 typedef typename ParallelIndexSet::const_iterator IndexIterator;
01045
01046 IndexIterator iEnd = indexSet_.end();
01047 IndexIterator index = indexSet_.begin();
01048 int bpos = 0;
01049 int publish;
01050
01051 assert(checkReset());
01052
01053 MPI_Status status;
01054
01055
01056 MPI_Probe(MPI_ANY_SOURCE, 345, remoteIndices_.communicator(), &status);
01057
01058 int source=status.MPI_SOURCE;
01059 int count;
01060 MPI_Get_count(&status, MPI_PACKED, &count);
01061
01062 Dune::dvverb<<rank_<<": Receiving message from "<< source<<" with "<<count<<" bytes"<<std::endl;
01063
01064 if(count>receiveBufferSize_) {
01065 receiveBufferSize_=count;
01066 delete[] receiveBuffer_;
01067 receiveBuffer_ = new char[receiveBufferSize_];
01068 }
01069
01070 MPI_Recv(receiveBuffer_, count, MPI_PACKED, source, 345, remoteIndices_.communicator(), &status);
01071
01072
01073 MPI_Unpack(receiveBuffer_, count, &bpos, &publish, 1, MPI_INT, remoteIndices_.communicator());
01074
01075
01076 while(publish>0) {
01077
01078
01079 GlobalIndex global;
01080 char sourceAttribute;
01081 int pairs;
01082
01083 MPI_Unpack(receiveBuffer_, count, &bpos, &global, 1, MPITraits<GlobalIndex>::getType(),
01084 remoteIndices_.communicator());
01085 MPI_Unpack(receiveBuffer_, count, &bpos, &sourceAttribute, 1, MPI_CHAR,
01086 remoteIndices_.communicator());
01087 MPI_Unpack(receiveBuffer_, count, &bpos, &pairs, 1, MPI_INT,
01088 remoteIndices_.communicator());
01089
01090
01091
01092 SLList<std::pair<int,Attribute> > sourceAttributeList;
01093 sourceAttributeList.push_back(std::make_pair(source,Attribute(sourceAttribute)));
01094 #ifndef NDEBUG
01095 bool foundSelf = false;
01096 #endif
01097 Attribute myAttribute=Attribute();
01098
01099
01100 for(; pairs>0; --pairs) {
01101
01102 int process;
01103 char attribute;
01104 MPI_Unpack(receiveBuffer_, count, &bpos, &process, 1, MPI_INT,
01105 remoteIndices_.communicator());
01106
01107 MPI_Unpack(receiveBuffer_, count, &bpos, &attribute, 1, MPI_CHAR,
01108 remoteIndices_.communicator());
01109
01110 if(process==rank_) {
01111 #ifndef NDEBUG
01112 foundSelf=true;
01113 #endif
01114 myAttribute=Attribute(attribute);
01115
01116
01117
01118 IndexIterator pos = std::lower_bound(index, iEnd, IndexPair(global));
01119
01120 if(pos == iEnd || pos->global() != global) {
01121
01122 indexSet_.add(global,
01123 ParallelLocalIndex<Attribute>(numberer(global),
01124 myAttribute, true));
01125 Dune::dvverb << "Adding "<<global<<" "<<myAttribute<<std::endl;
01126 continue;
01127 }
01128
01129
01130 bool indexIsThere = false;
01131 index=pos;
01132
01133 for(; pos->global()==global; ++pos)
01134 if(pos->local().attribute() == myAttribute) {
01135 Dune::dvverb<<"found "<<global<<" "<<myAttribute<<std::endl;
01136 indexIsThere = true;
01137 break;
01138 }
01139
01140 if(!indexIsThere) {
01141 indexSet_.add(global,
01142 ParallelLocalIndex<Attribute>(numberer(global),
01143 myAttribute, true));
01144 Dune::dvverb << "Adding "<<global<<" "<<myAttribute<<std::endl;
01145 }
01146
01147 }else{
01148 sourceAttributeList.push_back(std::make_pair(process,Attribute(attribute)));
01149 }
01150 }
01151 assert(foundSelf);
01152
01153 typedef typename SLList<std::pair<int,Attribute> >::const_iterator Iter;
01154 for(Iter i=sourceAttributeList.begin(), end=sourceAttributeList.end();
01155 i!=end; ++i)
01156 insertIntoRemoteIndexList(i->first, std::make_pair(global, myAttribute),
01157 i->second);
01158 --publish;
01159 }
01160
01161 resetIteratorsMap();
01162 }
01163
01164 template<typename T>
01165 void IndicesSyncer<T>::resetIteratorsMap(){
01166
01167
01168 typedef typename IteratorsMap::iterator Iterator;
01169 typedef typename RemoteIndices::RemoteIndexMap::iterator
01170 RemoteIterator;
01171 typedef typename GlobalIndicesMap::iterator GlobalIterator;
01172 typedef typename BoolMap::iterator BoolIterator;
01173
01174 const RemoteIterator remoteEnd = remoteIndices_.remoteIndices_.end();
01175 Iterator iterators = iteratorsMap_.begin();
01176 GlobalIterator global = globalMap_.begin();
01177 BoolIterator added = oldMap_.begin();
01178
01179 for(RemoteIterator remote = remoteIndices_.remoteIndices_.begin();
01180 remote != remoteEnd; ++remote, ++global, ++added, ++iterators) {
01181 iterators->second.reset(*(remote->second.first), global->second, added->second);
01182 }
01183 }
01184
01185 template<typename T>
01186 bool IndicesSyncer<T>::checkReset(const Iterators& iterators, RemoteIndexList& rList, GlobalIndexList& gList,
01187 BoolList& bList){
01188
01189 if(std::get<0>(iterators.iterators_) != rList.begin())
01190 return false;
01191 if(std::get<1>(iterators.iterators_) != gList.begin())
01192 return false;
01193 if(std::get<2>(iterators.iterators_) != bList.begin())
01194 return false;
01195 return true;
01196 }
01197
01198
01199 template<typename T>
01200 bool IndicesSyncer<T>::checkReset(){
01201
01202
01203 typedef typename IteratorsMap::iterator Iterator;
01204 typedef typename RemoteIndices::RemoteIndexMap::iterator
01205 RemoteIterator;
01206 typedef typename GlobalIndicesMap::iterator GlobalIterator;
01207 typedef typename BoolMap::iterator BoolIterator;
01208
01209 const RemoteIterator remoteEnd = remoteIndices_.remoteIndices_.end();
01210 Iterator iterators = iteratorsMap_.begin();
01211 GlobalIterator global = globalMap_.begin();
01212 BoolIterator added = oldMap_.begin();
01213 bool ret = true;
01214
01215 for(RemoteIterator remote = remoteIndices_.remoteIndices_.begin();
01216 remote != remoteEnd; ++remote, ++global, ++added, ++iterators) {
01217 if(!checkReset(iterators->second, *(remote->second.first), global->second,
01218 added->second))
01219 ret=false;
01220 }
01221 return ret;
01222 }
01223 }
01224
01225 #endif
01226 #endif