00001
00002
00003 #ifndef DUNE_REMOTEINDICES_HH
00004 #define DUNE_REMOTEINDICES_HH
00005
00006 #include "indexset.hh"
00007 #include "plocalindex.hh"
00008 #include <dune/common/exceptions.hh>
00009 #include <dune/common/poolallocator.hh>
00010 #include <dune/common/sllist.hh>
00011 #include <dune/common/stdstreams.hh>
00012 #include <map>
00013 #include <set>
00014 #include <utility>
00015 #include <iostream>
00016 #include <algorithm>
00017 #include <iterator>
00018 #if HAVE_MPI
00019 #include "mpitraits.hh"
00020 #include <mpi.h>
00021
00022 namespace Dune {
00033
00034 template<typename TG, typename TA>
00035 class MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >
00036 {
00037 public:
00038 inline static MPI_Datatype getType();
00039 private:
00040 static MPI_Datatype type;
00041 };
00042
00043
00044 template<typename T, typename A>
00045 class RemoteIndices;
00046
00047 template<typename T1, typename T2>
00048 class RemoteIndex;
00049
00050 template<typename T>
00051 class IndicesSyncer;
00052
00053 template<typename T1, typename T2>
00054 std::ostream& operator<<(std::ostream& os, const RemoteIndex<T1,T2>& index);
00055
00056
00057 template<typename T, typename A, bool mode>
00058 class RemoteIndexListModifier;
00059
00060
00064 template<typename T1, typename T2>
00065 class RemoteIndex
00066 {
00067 template<typename T>
00068 friend class IndicesSyncer;
00069
00070 template<typename T, typename A, typename A1>
00071 friend void repairLocalIndexPointers(std::map<int,SLList<std::pair<typename T::GlobalIndex, typename T::LocalIndex::Attribute>,A> >&,
00072 RemoteIndices<T,A1>&,
00073 const T&);
00074
00075 template<typename T, typename A, bool mode>
00076 friend class RemoteIndexListModifier;
00077
00078 public:
00083 typedef T1 GlobalIndex;
00092 typedef T2 Attribute;
00093
00097 typedef IndexPair<GlobalIndex,ParallelLocalIndex<Attribute> >
00098 PairType;
00099
00104 const Attribute attribute() const;
00105
00111 const PairType& localIndexPair() const;
00112
00116 RemoteIndex();
00117
00118
00124 RemoteIndex(const T2& attribute,
00125 const PairType* local);
00126
00127
00133 RemoteIndex(const T2& attribute);
00134
00135 bool operator==(const RemoteIndex& ri) const;
00136
00137 bool operator!=(const RemoteIndex& ri) const;
00138 private:
00140 const PairType* localIndex_;
00141
00143 char attribute_;
00144 };
00145
00146 template<class T, class A>
00147 std::ostream& operator<<(std::ostream& os, const RemoteIndices<T,A>& indices);
00148
00149 class InterfaceBuilder;
00150
00151 template<class T, class A>
00152 class CollectiveIterator;
00153
00154 template<class T>
00155 class IndicesSyncer;
00156
00157
00158 template<typename T1, typename T2>
00159 class OwnerOverlapCopyCommunication;
00160
00161
00178 template<class T, class A=std::allocator<RemoteIndex<typename T::GlobalIndex,
00179 typename T::LocalIndex::Attribute> > >
00180 class RemoteIndices
00181 {
00182 friend class InterfaceBuilder;
00183 friend class IndicesSyncer<T>;
00184 template<typename T1, typename A2, typename A1>
00185 friend void repairLocalIndexPointers(std::map<int,SLList<std::pair<typename T1::GlobalIndex, typename T1::LocalIndex::Attribute>,A2> >&,
00186 RemoteIndices<T1,A1>&,
00187 const T1&);
00188
00189 template<class G, class T1, class T2>
00190 friend void fillIndexSetHoles(const G& graph, Dune::OwnerOverlapCopyCommunication<T1,T2>& oocomm);
00191 friend std::ostream& operator<<<>(std::ostream&, const RemoteIndices<T>&);
00192
00193 public:
00194
00198 typedef T ParallelIndexSet;
00199
00202 typedef CollectiveIterator<T,A> CollectiveIteratorT;
00203
00207 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00208
00209
00213 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00214
00218 typedef typename LocalIndex::Attribute Attribute;
00219
00223 typedef Dune::RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00224
00225
00229 typedef typename A::template rebind<RemoteIndex>::other Allocator;
00230
00232 typedef Dune::SLList<RemoteIndex,Allocator>
00233 RemoteIndexList;
00234
00236 typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00237 RemoteIndexMap;
00238
00239 typedef typename RemoteIndexMap::const_iterator const_iterator;
00240
00258 inline RemoteIndices(const ParallelIndexSet& source, const ParallelIndexSet& destination,
00259 const MPI_Comm& comm, const std::vector<int>& neighbours=std::vector<int>(), bool includeSelf=false);
00260
00261 RemoteIndices();
00262
00270 void setIncludeSelf(bool includeSelf);
00271
00288 void setIndexSets(const ParallelIndexSet& source, const ParallelIndexSet& destination,
00289 const MPI_Comm& comm, const std::vector<int>& neighbours=std::vector<int>());
00290
00291 template<typename C>
00292 void setNeighbours(const C& neighbours)
00293 {
00294 neighbourIds.clear();
00295 neighbourIds.insert(neighbours.begin(), neighbours.end());
00296
00297 }
00298
00299 const std::set<int>& getNeighbours() const
00300 {
00301 return neighbourIds;
00302 }
00303
00307 ~RemoteIndices();
00308
00318 template<bool ignorePublic>
00319 void rebuild();
00320
00321 bool operator==(const RemoteIndices& ri);
00322
00330 inline bool isSynced() const;
00331
00335 inline MPI_Comm communicator() const;
00336
00351 template<bool mode, bool send>
00352 inline RemoteIndexListModifier<T,A,mode> getModifier(int process);
00353
00360 inline const_iterator find(int proc) const;
00361
00366 inline const_iterator begin() const;
00367
00372 inline const_iterator end() const;
00373
00377 template<bool send>
00378 inline CollectiveIteratorT iterator() const;
00379
00383 inline void free();
00384
00389 inline int neighbours() const;
00390
00392 inline const ParallelIndexSet& sourceIndexSet() const;
00393
00395 inline const ParallelIndexSet& destinationIndexSet() const;
00396
00397 private:
00399 RemoteIndices(const RemoteIndices&)
00400 {}
00401
00403 const ParallelIndexSet* source_;
00404
00406 const ParallelIndexSet* target_;
00407
00409 MPI_Comm comm_;
00410
00413 std::set<int> neighbourIds;
00414
00416 const static int commTag_=333;
00417
00422 int sourceSeqNo_;
00423
00428 int destSeqNo_;
00429
00433 bool publicIgnored;
00434
00438 bool firstBuild;
00439
00440
00441
00442
00443
00444
00445
00446 bool includeSelf;
00447
00449 typedef IndexPair<GlobalIndex, LocalIndex>
00450 PairType;
00451
00458 RemoteIndexMap remoteIndices_;
00459
00470 template<bool ignorePublic>
00471 inline void buildRemote(bool includeSelf);
00472
00478 inline int noPublic(const ParallelIndexSet& indexSet);
00479
00491 template<bool ignorePublic>
00492 inline void packEntries(PairType** myPairs, const ParallelIndexSet& indexSet,
00493 char* p_out, MPI_Datatype type, int bufferSize,
00494 int* position, int n);
00495
00509 inline void unpackIndices(RemoteIndexList& remote, int remoteEntries,
00510 PairType** local, int localEntries, char* p_in,
00511 MPI_Datatype type, int* position, int bufferSize,
00512 bool fromOurself);
00513
00514 inline void unpackIndices(RemoteIndexList& send, RemoteIndexList& receive,
00515 int remoteEntries, PairType** localSource,
00516 int localSourceEntries, PairType** localDest,
00517 int localDestEntries, char* p_in,
00518 MPI_Datatype type, int* position, int bufferSize);
00519
00520 void unpackCreateRemote(char* p_in, PairType** sourcePairs, PairType** DestPairs,
00521 int remoteProc, int sourcePublish, int destPublish,
00522 int bufferSize, bool sendTwo, bool fromOurSelf=false);
00523 };
00524
00542 template<class T, class A, bool mode>
00543 class RemoteIndexListModifier
00544 {
00545
00546 template<typename T1, typename A1>
00547 friend class RemoteIndices;
00548
00549 public:
00550 class InvalidPosition : public RangeError
00551 {};
00552
00553 enum {
00562 MODIFYINDEXSET=mode
00563 };
00564
00568 typedef T ParallelIndexSet;
00569
00573 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00574
00578 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00579
00583 typedef typename LocalIndex::Attribute Attribute;
00584
00588 typedef Dune::RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00589
00593 typedef A Allocator;
00594
00596 typedef Dune::SLList<RemoteIndex,Allocator>
00597 RemoteIndexList;
00598
00602 typedef SLListModifyIterator<RemoteIndex,Allocator> ModifyIterator;
00603
00607 typedef typename RemoteIndexList::const_iterator ConstIterator;
00608
00622 void insert(const RemoteIndex& index) throw(InvalidPosition);
00623
00624
00639 void insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition);
00640
00648 bool remove(const GlobalIndex& global) throw(InvalidPosition);
00649
00662 void repairLocalIndexPointers() throw(InvalidIndexSetState);
00663
00664
00665 RemoteIndexListModifier(const RemoteIndexListModifier&);
00666
00671 RemoteIndexListModifier()
00672 : glist_()
00673 {}
00674
00675 private:
00676
00682 RemoteIndexListModifier(const ParallelIndexSet& indexSet,
00683 RemoteIndexList& rList);
00684
00685 typedef SLList<GlobalIndex,Allocator> GlobalList;
00686 typedef typename GlobalList::ModifyIterator GlobalModifyIterator;
00687 RemoteIndexList* rList_;
00688 const ParallelIndexSet* indexSet_;
00689 GlobalList glist_;
00690 ModifyIterator iter_;
00691 GlobalModifyIterator giter_;
00692 ConstIterator end_;
00693 bool first_;
00694 GlobalIndex last_;
00695 };
00696
00701 template<class T, class A>
00702 class CollectiveIterator
00703 {
00704
00708 typedef T ParallelIndexSet;
00709
00713 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00714
00718 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00719
00723 typedef typename LocalIndex::Attribute Attribute;
00724
00726 typedef Dune::RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00727
00729 typedef typename A::template rebind<RemoteIndex>::other Allocator;
00730
00732 typedef Dune::SLList<RemoteIndex,Allocator> RemoteIndexList;
00733
00735 typedef std::map<int,std::pair<typename RemoteIndexList::const_iterator,
00736 const typename RemoteIndexList::const_iterator> >
00737 Map;
00738
00739 public:
00740
00742 typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00743 RemoteIndexMap;
00744
00750 inline CollectiveIterator(const RemoteIndexMap& map_, bool send);
00751
00760 inline void advance(const GlobalIndex& global);
00761
00771 inline void advance(const GlobalIndex& global, const Attribute& attribute);
00772
00773 CollectiveIterator& operator++();
00774
00778 inline bool empty();
00779
00786 class iterator
00787 {
00788 public:
00789 typedef typename Map::iterator RealIterator;
00790 typedef typename Map::iterator ConstRealIterator;
00791
00792
00794 iterator(const RealIterator& iter, const ConstRealIterator& end, GlobalIndex& index)
00795 : iter_(iter), end_(end), index_(index), hasAttribute(false)
00796 {
00797
00798 while(iter_!=end_ && iter_->second.first->localIndexPair().global()!=index_)
00799 ++iter_;
00800 }
00801
00802 iterator(const RealIterator& iter, const ConstRealIterator& end, GlobalIndex index,
00803 Attribute attribute)
00804 : iter_(iter), end_(end), index_(index), attribute_(attribute), hasAttribute(true)
00805 {
00806
00807 while(iter_!=end_ && (iter_->second.first->localIndexPair().global()!=index_
00808 || iter_->second.first->localIndexPair().local().attribute()!=attribute))
00809 ++iter_;
00810 }
00812 iterator(const iterator& other)
00813 : iter_(other.iter_), end_(other.end_), index_(other.index_)
00814 { }
00815
00817 iterator& operator++()
00818 {
00819 ++iter_;
00820
00821 while(iter_!=end_ && (iter_->second.first->localIndexPair().global()!=index_ ||
00822 (hasAttribute &&
00823 iter_->second.first->localIndexPair().local().attribute()!=attribute_)))
00824 ++iter_;
00825 assert(iter_==end_ ||
00826 (iter_->second.first->localIndexPair().global()==index_));
00827 assert(iter_==end_ || !hasAttribute ||
00828 (iter_->second.first->localIndexPair().local().attribute()==attribute_));
00829 return *this;
00830 }
00831
00833 const RemoteIndex& operator*() const
00834 {
00835 return *(iter_->second.first);
00836 }
00837
00839 int process() const
00840 {
00841 return iter_->first;
00842 }
00843
00845 const RemoteIndex* operator->() const
00846 {
00847 return iter_->second.first.operator->();
00848 }
00849
00851 bool operator==(const iterator& other)
00852 {
00853 return other.iter_==iter_;
00854 }
00855
00857 bool operator!=(const iterator& other)
00858 {
00859 return other.iter_!=iter_;
00860 }
00861
00862 private:
00863 iterator();
00864
00865 RealIterator iter_;
00866 RealIterator end_;
00867 GlobalIndex index_;
00868 Attribute attribute_;
00869 bool hasAttribute;
00870 };
00871
00872 iterator begin();
00873
00874 iterator end();
00875
00876 private:
00877
00878 Map map_;
00879 GlobalIndex index_;
00880 Attribute attribute_;
00881 bool noattribute;
00882 };
00883
00884 template<typename TG, typename TA>
00885 MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::getType()
00886 {
00887 if(type==MPI_DATATYPE_NULL) {
00888 int length[2] = {1, 1};
00889 MPI_Aint base;
00890 MPI_Aint disp[2];
00891 MPI_Datatype types[2] = {MPITraits<TG>::getType(),
00892 MPITraits<ParallelLocalIndex<TA> >::getType()};
00893 IndexPair<TG,ParallelLocalIndex<TA> > rep;
00894 MPI_Get_address(&rep, &base);
00895 MPI_Get_address(&(rep.global_), &disp[0]);
00896 MPI_Get_address(&(rep.local_), &disp[1]);
00897 for (MPI_Aint& d : disp)
00898 d -= base;
00899
00900 MPI_Datatype tmp;
00901 MPI_Type_create_struct(2, length, disp, types, &tmp);
00902
00903 MPI_Type_create_resized(tmp, 0, sizeof(IndexPair<TG,ParallelLocalIndex<TA> >), &type);
00904 MPI_Type_commit(&type);
00905
00906 MPI_Type_free(&tmp);
00907 }
00908 return type;
00909 }
00910
00911 template<typename TG, typename TA>
00912 MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::type=MPI_DATATYPE_NULL;
00913
00914 template<typename T1, typename T2>
00915 RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute, const PairType* local)
00916 : localIndex_(local), attribute_(attribute)
00917 {}
00918
00919 template<typename T1, typename T2>
00920 RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute)
00921 : localIndex_(0), attribute_(attribute)
00922 {}
00923
00924 template<typename T1, typename T2>
00925 RemoteIndex<T1,T2>::RemoteIndex()
00926 : localIndex_(0), attribute_()
00927 {}
00928 template<typename T1, typename T2>
00929 inline bool RemoteIndex<T1,T2>::operator==(const RemoteIndex& ri) const
00930 {
00931 return localIndex_==ri.localIndex_ && attribute_==ri.attribute;
00932 }
00933
00934 template<typename T1, typename T2>
00935 inline bool RemoteIndex<T1,T2>::operator!=(const RemoteIndex& ri) const
00936 {
00937 return localIndex_!=ri.localIndex_ || attribute_!=ri.attribute_;
00938 }
00939
00940 template<typename T1, typename T2>
00941 inline const T2 RemoteIndex<T1,T2>::attribute() const
00942 {
00943 return T2(attribute_);
00944 }
00945
00946 template<typename T1, typename T2>
00947 inline const IndexPair<T1,ParallelLocalIndex<T2> >& RemoteIndex<T1,T2>::localIndexPair() const
00948 {
00949 return *localIndex_;
00950 }
00951
00952 template<typename T, typename A>
00953 inline RemoteIndices<T,A>::RemoteIndices(const ParallelIndexSet& source,
00954 const ParallelIndexSet& destination,
00955 const MPI_Comm& comm,
00956 const std::vector<int>& neighbours,
00957 bool includeSelf_)
00958 : source_(&source), target_(&destination), comm_(comm),
00959 sourceSeqNo_(-1), destSeqNo_(-1), publicIgnored(false), firstBuild(true),
00960 includeSelf(includeSelf_)
00961 {
00962 setNeighbours(neighbours);
00963 }
00964
00965 template<typename T, typename A>
00966 void RemoteIndices<T,A>::setIncludeSelf(bool b)
00967 {
00968 includeSelf=b;
00969 }
00970
00971 template<typename T, typename A>
00972 RemoteIndices<T,A>::RemoteIndices()
00973 : source_(0), target_(0), sourceSeqNo_(-1),
00974 destSeqNo_(-1), publicIgnored(false), firstBuild(true),
00975 includeSelf(false)
00976 {}
00977
00978 template<class T, typename A>
00979 void RemoteIndices<T,A>::setIndexSets(const ParallelIndexSet& source,
00980 const ParallelIndexSet& destination,
00981 const MPI_Comm& comm,
00982 const std::vector<int>& neighbours)
00983 {
00984 free();
00985 source_ = &source;
00986 target_ = &destination;
00987 comm_ = comm;
00988 firstBuild = true;
00989 setNeighbours(neighbours);
00990 }
00991
00992 template<typename T, typename A>
00993 const typename RemoteIndices<T,A>::ParallelIndexSet&
00994 RemoteIndices<T,A>::sourceIndexSet() const
00995 {
00996 return *source_;
00997 }
00998
00999
01000 template<typename T, typename A>
01001 const typename RemoteIndices<T,A>::ParallelIndexSet&
01002 RemoteIndices<T,A>::destinationIndexSet() const
01003 {
01004 return *target_;
01005 }
01006
01007
01008 template<typename T, typename A>
01009 RemoteIndices<T,A>::~RemoteIndices()
01010 {
01011 free();
01012 }
01013
01014 template<typename T, typename A>
01015 template<bool ignorePublic>
01016 inline void RemoteIndices<T,A>::packEntries(IndexPair<GlobalIndex,LocalIndex>** pairs,
01017 const ParallelIndexSet& indexSet,
01018 char* p_out, MPI_Datatype type,
01019 int bufferSize,
01020 int *position, int n)
01021 {
01022 DUNE_UNUSED_PARAMETER(n);
01023
01024 typedef typename ParallelIndexSet::const_iterator const_iterator;
01025 typedef IndexPair<GlobalIndex,LocalIndex> PairType;
01026 const const_iterator end = indexSet.end();
01027
01028
01029 int i=0;
01030 for(const_iterator index = indexSet.begin(); index != end; ++index)
01031 if(ignorePublic || index->local().isPublic()) {
01032
01033 MPI_Pack(const_cast<PairType*>(&(*index)), 1,
01034 type,
01035 p_out, bufferSize, position, comm_);
01036 pairs[i++] = const_cast<PairType*>(&(*index));
01037
01038 }
01039 assert(i==n);
01040 }
01041
01042 template<typename T, typename A>
01043 inline int RemoteIndices<T,A>::noPublic(const ParallelIndexSet& indexSet)
01044 {
01045 typedef typename ParallelIndexSet::const_iterator const_iterator;
01046
01047 int noPublic=0;
01048
01049 const const_iterator end=indexSet.end();
01050 for(const_iterator index=indexSet.begin(); index!=end; ++index)
01051 if(index->local().isPublic())
01052 noPublic++;
01053
01054 return noPublic;
01055
01056 }
01057
01058
01059 template<typename T, typename A>
01060 inline void RemoteIndices<T,A>::unpackCreateRemote(char* p_in, PairType** sourcePairs,
01061 PairType** destPairs, int remoteProc,
01062 int sourcePublish, int destPublish,
01063 int bufferSize, bool sendTwo,
01064 bool fromOurSelf)
01065 {
01066
01067
01068 int noRemoteSource=-1, noRemoteDest=-1;
01069 char twoIndexSets=0;
01070 int position=0;
01071
01072 MPI_Unpack(p_in, bufferSize, &position, &twoIndexSets, 1, MPI_CHAR, comm_);
01073
01074 MPI_Unpack(p_in, bufferSize, &position, &noRemoteSource, 1, MPI_INT, comm_);
01075
01076 MPI_Unpack(p_in, bufferSize, &position, &noRemoteDest, 1, MPI_INT, comm_);
01077
01078
01079
01080 RemoteIndexList* receive= new RemoteIndexList();
01081
01082 RemoteIndexList* send=0;
01083
01084 MPI_Datatype type= MPITraits<PairType>::getType();
01085
01086 if(!twoIndexSets) {
01087 if(sendTwo) {
01088 send = new RemoteIndexList();
01089
01090 unpackIndices(*send, *receive, noRemoteSource, sourcePairs, sourcePublish,
01091 destPairs, destPublish, p_in, type, &position, bufferSize);
01092 }else{
01093
01094 unpackIndices(*receive, noRemoteSource, sourcePairs, sourcePublish,
01095 p_in, type, &position, bufferSize, fromOurSelf);
01096 send=receive;
01097 }
01098 }else{
01099
01100 int oldPos=position;
01101
01102 unpackIndices(*receive, noRemoteSource, destPairs, destPublish,
01103 p_in, type, &position, bufferSize, fromOurSelf);
01104 if(!sendTwo)
01105
01106 position=oldPos;
01107
01108 send = new RemoteIndexList();
01109 unpackIndices(*send, noRemoteDest, sourcePairs, sourcePublish,
01110 p_in, type, &position, bufferSize, fromOurSelf);
01111 }
01112
01113 if(receive->empty() && send->empty()) {
01114 if(send==receive) {
01115 delete send;
01116 }else{
01117 delete send;
01118 delete receive;
01119 }
01120 }else{
01121 remoteIndices_.insert(std::make_pair(remoteProc,
01122 std::make_pair(send,receive)));
01123 }
01124 }
01125
01126
01127 template<typename T, typename A>
01128 template<bool ignorePublic>
01129 inline void RemoteIndices<T,A>::buildRemote(bool includeSelf_)
01130 {
01131
01132 int rank, procs;
01133 MPI_Comm_rank(comm_, &rank);
01134 MPI_Comm_size(comm_, &procs);
01135
01136
01137
01138 int sourcePublish, destPublish;
01139
01140
01141 char sendTwo = (source_ != target_);
01142
01143 if(procs==1 && !(sendTwo || includeSelf_))
01144
01145 return;
01146
01147 sourcePublish = (ignorePublic) ? source_->size() : noPublic(*source_);
01148
01149 if(sendTwo)
01150 destPublish = (ignorePublic) ? target_->size() : noPublic(*target_);
01151 else
01152
01153 destPublish = 0;
01154
01155 int maxPublish, publish=sourcePublish+destPublish;
01156
01157
01158 MPI_Allreduce(&publish, &maxPublish, 1, MPI_INT, MPI_MAX, comm_);
01159
01160
01161 typedef IndexPair<GlobalIndex,LocalIndex> PairType;
01162
01163 PairType** destPairs;
01164 PairType** sourcePairs = new PairType*[sourcePublish>0 ? sourcePublish : 1];
01165
01166 if(sendTwo)
01167 destPairs = new PairType*[destPublish>0 ? destPublish : 1];
01168 else
01169 destPairs=sourcePairs;
01170
01171 char** buffer = new char*[2];
01172 int bufferSize;
01173 int position=0;
01174 int intSize;
01175 int charSize;
01176
01177
01178 MPI_Datatype type = MPITraits<PairType>::getType();
01179
01180 MPI_Pack_size(maxPublish, type, comm_,
01181 &bufferSize);
01182 MPI_Pack_size(1, MPI_INT, comm_,
01183 &intSize);
01184 MPI_Pack_size(1, MPI_CHAR, comm_,
01185 &charSize);
01186
01187
01188
01189
01190 bufferSize += 2 * intSize + charSize;
01191
01192 if(bufferSize<=0) bufferSize=1;
01193
01194 buffer[0] = new char[bufferSize];
01195 buffer[1] = new char[bufferSize];
01196
01197
01198
01199 MPI_Pack(&sendTwo, 1, MPI_CHAR, buffer[0], bufferSize, &position,
01200 comm_);
01201
01202
01203 MPI_Pack(&sourcePublish, 1, MPI_INT, buffer[0], bufferSize, &position,
01204 comm_);
01205 MPI_Pack(&destPublish, 1, MPI_INT, buffer[0], bufferSize, &position,
01206 comm_);
01207
01208
01209 packEntries<ignorePublic>(sourcePairs, *source_, buffer[0], type,
01210 bufferSize, &position, sourcePublish);
01211
01212 if(sendTwo)
01213 packEntries<ignorePublic>(destPairs, *target_, buffer[0], type,
01214 bufferSize, &position, destPublish);
01215
01216
01217
01218 if(sendTwo|| includeSelf_)
01219 unpackCreateRemote(buffer[0], sourcePairs, destPairs, rank, sourcePublish,
01220 destPublish, bufferSize, sendTwo, includeSelf_);
01221
01222 neighbourIds.erase(rank);
01223
01224 if(neighbourIds.size()==0)
01225 {
01226 Dune::dvverb<<rank<<": Sending messages in a ring"<<std::endl;
01227
01228 for(int proc=1; proc<procs; proc++) {
01229
01230 char* p_out = buffer[1-(proc%2)];
01231 char* p_in = buffer[proc%2];
01232
01233 MPI_Status status;
01234 if(rank%2==0) {
01235 MPI_Ssend(p_out, bufferSize, MPI_PACKED, (rank+1)%procs,
01236 commTag_, comm_);
01237 MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
01238 commTag_, comm_, &status);
01239 }else{
01240 MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
01241 commTag_, comm_, &status);
01242 MPI_Ssend(p_out, bufferSize, MPI_PACKED, (rank+1)%procs,
01243 commTag_, comm_);
01244 }
01245
01246
01247
01248 int remoteProc = (rank+procs-proc)%procs;
01249
01250 unpackCreateRemote(p_in, sourcePairs, destPairs, remoteProc, sourcePublish,
01251 destPublish, bufferSize, sendTwo);
01252
01253 }
01254
01255 }
01256 else
01257 {
01258 MPI_Request* requests=new MPI_Request[neighbourIds.size()];
01259 MPI_Request* req=requests;
01260
01261 typedef typename std::set<int>::size_type size_type;
01262 size_type noNeighbours=neighbourIds.size();
01263
01264
01265 for(std::set<int>::iterator neighbour=neighbourIds.begin();
01266 neighbour!= neighbourIds.end(); ++neighbour) {
01267
01268 MPI_Issend(buffer[0], position , MPI_PACKED, *neighbour, commTag_, comm_, req++);
01269 }
01270
01271
01272
01273 for(size_type received=0; received <noNeighbours; ++received)
01274 {
01275 MPI_Status status;
01276
01277 MPI_Probe(MPI_ANY_SOURCE, commTag_, comm_, &status);
01278 int remoteProc=status.MPI_SOURCE;
01279 int size;
01280 MPI_Get_count(&status, MPI_PACKED, &size);
01281
01282 MPI_Recv(buffer[1], size, MPI_PACKED, remoteProc,
01283 commTag_, comm_, &status);
01284
01285 unpackCreateRemote(buffer[1], sourcePairs, destPairs, remoteProc, sourcePublish,
01286 destPublish, bufferSize, sendTwo);
01287 }
01288
01289 MPI_Status* statuses = new MPI_Status[neighbourIds.size()];
01290
01291 if(MPI_ERR_IN_STATUS==MPI_Waitall(neighbourIds.size(), requests, statuses)) {
01292 for(size_type i=0; i < neighbourIds.size(); ++i)
01293 if(statuses[i].MPI_ERROR!=MPI_SUCCESS) {
01294 std::cerr<<rank<<": MPI_Error occurred while receiving message."<<std::endl;
01295 MPI_Abort(comm_, 999);
01296 }
01297 }
01298 delete[] requests;
01299 delete[] statuses;
01300 }
01301
01302
01303
01304 if(destPairs!=sourcePairs)
01305 delete[] destPairs;
01306
01307 delete[] sourcePairs;
01308 delete[] buffer[0];
01309 delete[] buffer[1];
01310 delete[] buffer;
01311 }
01312
01313 template<typename T, typename A>
01314 inline void RemoteIndices<T,A>::unpackIndices(RemoteIndexList& remote,
01315 int remoteEntries,
01316 PairType** local,
01317 int localEntries,
01318 char* p_in,
01319 MPI_Datatype type,
01320 int* position,
01321 int bufferSize,
01322 bool fromOurSelf)
01323 {
01324 if(remoteEntries==0)
01325 return;
01326
01327 PairType index(1);
01328 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01329 type, comm_);
01330 GlobalIndex oldGlobal=index.global();
01331 int n_in=0, localIndex=0;
01332
01333
01334 while(localIndex<localEntries) {
01335 if(local[localIndex]->global()==index.global()) {
01336 int oldLocalIndex=localIndex;
01337
01338 while(localIndex<localEntries &&
01339 local[localIndex]->global()==index.global()) {
01340 if(!fromOurSelf || index.local().attribute() !=
01341 local[localIndex]->local().attribute())
01342
01343 remote.push_back(RemoteIndex(index.local().attribute(),
01344 local[localIndex]));
01345 localIndex++;
01346 }
01347
01348
01349 if((++n_in) < remoteEntries) {
01350 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01351 type, comm_);
01352 if(index.global()==oldGlobal)
01353
01354 localIndex=oldLocalIndex;
01355 else
01356 oldGlobal=index.global();
01357 }else{
01358
01359 break;
01360 }
01361 continue;
01362 }
01363
01364 if (local[localIndex]->global()<index.global()) {
01365
01366 ++localIndex;
01367 }else{
01368
01369 if((++n_in) < remoteEntries) {
01370 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01371 type, comm_);
01372 oldGlobal=index.global();
01373 }else
01374
01375 break;
01376 }
01377 }
01378
01379
01380 while(++n_in < remoteEntries)
01381 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01382 type, comm_);
01383 }
01384
01385
01386 template<typename T, typename A>
01387 inline void RemoteIndices<T,A>::unpackIndices(RemoteIndexList& send,
01388 RemoteIndexList& receive,
01389 int remoteEntries,
01390 PairType** localSource,
01391 int localSourceEntries,
01392 PairType** localDest,
01393 int localDestEntries,
01394 char* p_in,
01395 MPI_Datatype type,
01396 int* position,
01397 int bufferSize)
01398 {
01399 int n_in=0, sourceIndex=0, destIndex=0;
01400
01401
01402 while(n_in<remoteEntries && (sourceIndex<localSourceEntries || destIndex<localDestEntries)) {
01403
01404 PairType index;
01405 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01406 type, comm_);
01407 n_in++;
01408
01409
01410 while(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()<index.global())
01411 sourceIndex++;
01412
01413 while(destIndex<localDestEntries && localDest[destIndex]->global()<index.global())
01414 destIndex++;
01415
01416
01417 if(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()==index.global())
01418 send.push_back(RemoteIndex(index.local().attribute(),
01419 localSource[sourceIndex]));
01420
01421 if(destIndex < localDestEntries && localDest[destIndex]->global() == index.global())
01422 receive.push_back(RemoteIndex(index.local().attribute(),
01423 localDest[sourceIndex]));
01424 }
01425
01426 }
01427
01428 template<typename T, typename A>
01429 inline void RemoteIndices<T,A>::free()
01430 {
01431 typedef typename RemoteIndexMap::iterator Iterator;
01432 Iterator lend = remoteIndices_.end();
01433 for(Iterator lists=remoteIndices_.begin(); lists != lend; ++lists) {
01434 if(lists->second.first==lists->second.second) {
01435
01436 delete lists->second.first;
01437 }else{
01438 delete lists->second.first;
01439 delete lists->second.second;
01440 }
01441 }
01442 remoteIndices_.clear();
01443 firstBuild=true;
01444 }
01445
01446 template<typename T, typename A>
01447 inline int RemoteIndices<T,A>::neighbours() const
01448 {
01449 return remoteIndices_.size();
01450 }
01451
01452 template<typename T, typename A>
01453 template<bool ignorePublic>
01454 inline void RemoteIndices<T,A>::rebuild()
01455 {
01456
01457 if(firstBuild ||
01458 ignorePublic!=publicIgnored || !
01459 isSynced()) {
01460 free();
01461
01462 buildRemote<ignorePublic>(includeSelf);
01463
01464 sourceSeqNo_ = source_->seqNo();
01465 destSeqNo_ = target_->seqNo();
01466 firstBuild=false;
01467 publicIgnored=ignorePublic;
01468 }
01469
01470
01471 }
01472
01473 template<typename T, typename A>
01474 inline bool RemoteIndices<T,A>::isSynced() const
01475 {
01476 return sourceSeqNo_==source_->seqNo() && destSeqNo_ ==target_->seqNo();
01477 }
01478
01479 template<typename T, typename A>
01480 template<bool mode, bool send>
01481 RemoteIndexListModifier<T,A,mode> RemoteIndices<T,A>::getModifier(int process)
01482 {
01483
01484
01485
01486
01487 sourceSeqNo_ = source_->seqNo();
01488 destSeqNo_ = target_->seqNo();
01489
01490 typename RemoteIndexMap::iterator found = remoteIndices_.find(process);
01491
01492 if(found == remoteIndices_.end())
01493 {
01494 if(source_ != target_)
01495 found = remoteIndices_.insert(found, std::make_pair(process,
01496 std::make_pair(new RemoteIndexList(),
01497 new RemoteIndexList())));
01498 else{
01499 RemoteIndexList* rlist = new RemoteIndexList();
01500 found = remoteIndices_.insert(found,
01501 std::make_pair(process,
01502 std::make_pair(rlist, rlist)));
01503 }
01504 }
01505
01506 firstBuild = false;
01507
01508 if(send)
01509 return RemoteIndexListModifier<T,A,mode>(*source_, *(found->second.first));
01510 else
01511 return RemoteIndexListModifier<T,A,mode>(*target_, *(found->second.second));
01512 }
01513
01514 template<typename T, typename A>
01515 inline typename RemoteIndices<T,A>::const_iterator
01516 RemoteIndices<T,A>::find(int proc) const
01517 {
01518 return remoteIndices_.find(proc);
01519 }
01520
01521 template<typename T, typename A>
01522 inline typename RemoteIndices<T,A>::const_iterator
01523 RemoteIndices<T,A>::begin() const
01524 {
01525 return remoteIndices_.begin();
01526 }
01527
01528 template<typename T, typename A>
01529 inline typename RemoteIndices<T,A>::const_iterator
01530 RemoteIndices<T,A>::end() const
01531 {
01532 return remoteIndices_.end();
01533 }
01534
01535
01536 template<typename T, typename A>
01537 bool RemoteIndices<T,A>::operator==(const RemoteIndices& ri)
01538 {
01539 if(neighbours()!=ri.neighbours())
01540 return false;
01541
01542 typedef RemoteIndexList RList;
01543 typedef typename std::map<int,std::pair<RList*,RList*> >::const_iterator const_iterator;
01544
01545 const const_iterator rend = remoteIndices_.end();
01546
01547 for(const_iterator rindex = remoteIndices_.begin(), rindex1=ri.remoteIndices_.begin(); rindex!=rend; ++rindex, ++rindex1) {
01548 if(rindex->first != rindex1->first)
01549 return false;
01550 if(*(rindex->second.first) != *(rindex1->second.first))
01551 return false;
01552 if(*(rindex->second.second) != *(rindex1->second.second))
01553 return false;
01554 }
01555 return true;
01556 }
01557
01558 template<class T, class A, bool mode>
01559 RemoteIndexListModifier<T,A,mode>::RemoteIndexListModifier(const ParallelIndexSet& indexSet,
01560 RemoteIndexList& rList)
01561 : rList_(&rList), indexSet_(&indexSet), iter_(rList.beginModify()), end_(rList.end()), first_(true)
01562 {
01563 if(MODIFYINDEXSET) {
01564 assert(indexSet_);
01565 for(ConstIterator iter=iter_; iter != end_; ++iter)
01566 glist_.push_back(iter->localIndexPair().global());
01567 giter_ = glist_.beginModify();
01568 }
01569 }
01570
01571 template<typename T, typename A, bool mode>
01572 RemoteIndexListModifier<T,A,mode>::RemoteIndexListModifier(const RemoteIndexListModifier<T,A,mode>& other)
01573 : rList_(other.rList_), indexSet_(other.indexSet_),
01574 glist_(other.glist_), iter_(other.iter_), giter_(other.giter_), end_(other.end_),
01575 first_(other.first_), last_(other.last_)
01576 {}
01577
01578 template<typename T, typename A, bool mode>
01579 inline void RemoteIndexListModifier<T,A,mode>::repairLocalIndexPointers() throw(InvalidIndexSetState)
01580 {
01581 if(MODIFYINDEXSET) {
01582
01583 #ifdef DUNE_ISTL_WITH_CHECKING
01584 if(indexSet_->state()!=GROUND)
01585 DUNE_THROW(InvalidIndexSetState, "Index has to be in ground mode for repairing pointers to indices");
01586 #endif
01587 typedef typename ParallelIndexSet::const_iterator IndexIterator;
01588 typedef typename GlobalList::const_iterator GlobalIterator;
01589 typedef typename RemoteIndexList::iterator Iterator;
01590 GlobalIterator giter = glist_.begin();
01591 IndexIterator index = indexSet_->begin();
01592
01593 for(Iterator iter=rList_->begin(); iter != end_; ++iter) {
01594 while(index->global()<*giter) {
01595 ++index;
01596 #ifdef DUNE_ISTL_WITH_CHECKING
01597 if(index == indexSet_->end())
01598 DUNE_THROW(InvalidPosition, "No such global index in set!");
01599 #endif
01600 }
01601
01602 #ifdef DUNE_ISTL_WITH_CHECKING
01603 if(index->global() != *giter)
01604 DUNE_THROW(InvalidPosition, "No such global index in set!");
01605 #endif
01606 iter->localIndex_ = &(*index);
01607 }
01608 }
01609 }
01610
01611 template<typename T, typename A, bool mode>
01612 inline void RemoteIndexListModifier<T,A,mode>::insert(const RemoteIndex& index) throw(InvalidPosition)
01613 {
01614 static_assert(!mode,"Not allowed if the mode indicates that new indices"
01615 "might be added to the underlying index set. Use "
01616 "insert(const RemoteIndex&, const GlobalIndex&) instead");
01617
01618 #ifdef DUNE_ISTL_WITH_CHECKING
01619 if(!first_ && index.localIndexPair().global()<last_)
01620 DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01621 #endif
01622
01623 while(iter_ != end_ && iter_->localIndexPair().global() < index.localIndexPair().global()) {
01624 ++iter_;
01625 }
01626
01627
01628 assert(iter_==end_ || iter_->localIndexPair().global() != index.localIndexPair().global());
01629 iter_.insert(index);
01630 last_ = index.localIndexPair().global();
01631 first_ = false;
01632 }
01633
01634 template<typename T, typename A, bool mode>
01635 inline void RemoteIndexListModifier<T,A,mode>::insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition)
01636 {
01637 static_assert(mode,"Not allowed if the mode indicates that no new indices"
01638 "might be added to the underlying index set. Use "
01639 "insert(const RemoteIndex&) instead");
01640 #ifdef DUNE_ISTL_WITH_CHECKING
01641 if(!first_ && global<last_)
01642 DUNE_THROW(InvalidPosition, "Modification of remote indices have to occur with ascending global index.");
01643 #endif
01644
01645 while(iter_ != end_ && *giter_ < global) {
01646 ++giter_;
01647 ++iter_;
01648 }
01649
01650
01651 assert(iter_->localIndexPair().global() != global);
01652 iter_.insert(index);
01653 giter_.insert(global);
01654
01655 last_ = global;
01656 first_ = false;
01657 }
01658
01659 template<typename T, typename A, bool mode>
01660 bool RemoteIndexListModifier<T,A,mode>::remove(const GlobalIndex& global) throw(InvalidPosition)
01661 {
01662 #ifdef DUNE_ISTL_WITH_CHECKING
01663 if(!first_ && global<last_)
01664 DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01665 #endif
01666
01667 bool found= false;
01668
01669 if(MODIFYINDEXSET) {
01670
01671 while(iter_!=end_ && *giter_< global) {
01672 ++giter_;
01673 ++iter_;
01674 }
01675 if(*giter_ == global) {
01676 giter_.remove();
01677 iter_.remove();
01678 found=true;
01679 }
01680 }else{
01681 while(iter_!=end_ && iter_->localIndexPair().global() < global)
01682 ++iter_;
01683
01684 if(iter_->localIndexPair().global()==global) {
01685 iter_.remove();
01686 found = true;
01687 }
01688 }
01689
01690 last_ = global;
01691 first_ = false;
01692 return found;
01693 }
01694
01695 template<typename T, typename A>
01696 template<bool send>
01697 inline typename RemoteIndices<T,A>::CollectiveIteratorT RemoteIndices<T,A>::iterator() const
01698 {
01699 return CollectiveIterator<T,A>(remoteIndices_, send);
01700 }
01701
01702 template<typename T, typename A>
01703 inline MPI_Comm RemoteIndices<T,A>::communicator() const
01704 {
01705 return comm_;
01706
01707 }
01708
01709 template<typename T, typename A>
01710 CollectiveIterator<T,A>::CollectiveIterator(const RemoteIndexMap& pmap, bool send)
01711 {
01712 typedef typename RemoteIndexMap::const_iterator const_iterator;
01713
01714 const const_iterator end=pmap.end();
01715 for(const_iterator process=pmap.begin(); process != end; ++process) {
01716 const RemoteIndexList* list = send ? process->second.first : process->second.second;
01717 typedef typename RemoteIndexList::const_iterator iterator;
01718 map_.insert(std::make_pair(process->first,
01719 std::pair<iterator, const iterator>(list->begin(), list->end())));
01720 }
01721 }
01722
01723 template<typename T, typename A>
01724 inline void CollectiveIterator<T,A>::advance(const GlobalIndex& index)
01725 {
01726 typedef typename Map::iterator iterator;
01727 typedef typename Map::const_iterator const_iterator;
01728 const const_iterator end = map_.end();
01729
01730 for(iterator iter = map_.begin(); iter != end;) {
01731
01732 typename RemoteIndexList::const_iterator current = iter->second.first;
01733 typename RemoteIndexList::const_iterator rend = iter->second.second;
01734 RemoteIndex remoteIndex;
01735 if(current != rend)
01736 remoteIndex = *current;
01737
01738 while(iter->second.first!=iter->second.second && iter->second.first->localIndexPair().global()<index)
01739 ++(iter->second.first);
01740
01741
01742 if(iter->second.first == iter->second.second)
01743 map_.erase(iter++);
01744 else{
01745 ++iter;
01746 }
01747 }
01748 index_=index;
01749 noattribute=true;
01750 }
01751
01752 template<typename T, typename A>
01753 inline void CollectiveIterator<T,A>::advance(const GlobalIndex& index,
01754 const Attribute& attribute)
01755 {
01756 typedef typename Map::iterator iterator;
01757 typedef typename Map::const_iterator const_iterator;
01758 const const_iterator end = map_.end();
01759
01760 for(iterator iter = map_.begin(); iter != end;) {
01761
01762 typename RemoteIndexList::const_iterator current = iter->second.first;
01763 typename RemoteIndexList::const_iterator rend = iter->second.second;
01764 RemoteIndex remoteIndex;
01765 if(current != rend)
01766 remoteIndex = *current;
01767
01768
01769 while(iter->second.first!=iter->second.second && iter->second.first->localIndexPair().global()<index)
01770 ++(iter->second.first);
01771
01772
01773 while(iter->second.first!=iter->second.second
01774 && iter->second.first->localIndexPair().global()==index
01775 && iter->second.first->localIndexPair().local().attribute()<attribute)
01776 ++(iter->second.first);
01777
01778
01779 if(iter->second.first == iter->second.second)
01780 map_.erase(iter++);
01781 else{
01782 ++iter;
01783 }
01784 }
01785 index_=index;
01786 attribute_=attribute;
01787 noattribute=false;
01788 }
01789
01790 template<typename T, typename A>
01791 inline CollectiveIterator<T,A>& CollectiveIterator<T,A>::operator++()
01792 {
01793 typedef typename Map::iterator iterator;
01794 typedef typename Map::const_iterator const_iterator;
01795 const const_iterator end = map_.end();
01796
01797 for(iterator iter = map_.begin(); iter != end;) {
01798
01799 typename RemoteIndexList::const_iterator current = iter->second.first;
01800 typename RemoteIndexList::const_iterator rend = iter->second.second;
01801
01802
01803 if(iter->second.first->localIndexPair().global()==index_ &&
01804 (noattribute || iter->second.first->localIndexPair().local().attribute() == attribute_))
01805 ++(iter->second.first);
01806
01807
01808 if(iter->second.first == iter->second.second)
01809 map_.erase(iter++);
01810 else{
01811 ++iter;
01812 }
01813 }
01814 return *this;
01815 }
01816
01817 template<typename T, typename A>
01818 inline bool CollectiveIterator<T,A>::empty()
01819 {
01820 return map_.empty();
01821 }
01822
01823 template<typename T, typename A>
01824 inline typename CollectiveIterator<T,A>::iterator
01825 CollectiveIterator<T,A>::begin()
01826 {
01827 if(noattribute)
01828 return iterator(map_.begin(), map_.end(), index_);
01829 else
01830 return iterator(map_.begin(), map_.end(), index_,
01831 attribute_);
01832 }
01833
01834 template<typename T, typename A>
01835 inline typename CollectiveIterator<T,A>::iterator
01836 CollectiveIterator<T,A>::end()
01837 {
01838 return iterator(map_.end(), map_.end(), index_);
01839 }
01840
01841 template<typename TG, typename TA>
01842 inline std::ostream& operator<<(std::ostream& os, const RemoteIndex<TG,TA>& index)
01843 {
01844 os<<"[global="<<index.localIndexPair().global()<<", remote attribute="<<index.attribute()<<" local attribute="<<index.localIndexPair().local().attribute()<<"]";
01845 return os;
01846 }
01847
01848 template<typename T, typename A>
01849 inline std::ostream& operator<<(std::ostream& os, const RemoteIndices<T,A>& indices)
01850 {
01851 int rank;
01852 MPI_Comm_rank(indices.comm_, &rank);
01853
01854 typedef typename RemoteIndices<T,A>::RemoteIndexList RList;
01855 typedef typename std::map<int,std::pair<RList*,RList*> >::const_iterator const_iterator;
01856
01857 const const_iterator rend = indices.remoteIndices_.end();
01858
01859 for(const_iterator rindex = indices.remoteIndices_.begin(); rindex!=rend; ++rindex) {
01860 os<<rank<<": Prozess "<<rindex->first<<":";
01861
01862 if(!rindex->second.first->empty()) {
01863 os<<" send:";
01864
01865 const typename RList::const_iterator send= rindex->second.first->end();
01866
01867 for(typename RList::const_iterator index = rindex->second.first->begin();
01868 index != send; ++index)
01869 os<<*index<<" ";
01870 os<<std::endl;
01871 }
01872 if(!rindex->second.second->empty()) {
01873 os<<rank<<": Prozess "<<rindex->first<<": "<<"receive: ";
01874
01875 for(const auto& index : *(rindex->second.second))
01876 os << index << " ";
01877 }
01878 os<<std::endl<<std::flush;
01879 }
01880 return os;
01881 }
01883 }
01884
01885 #endif
01886 #endif