Commit 4291a86d authored by BRAMAS Berenger's avatar BRAMAS Berenger

make it work for mpi and other stuff

parent 800ef298
......@@ -6,6 +6,15 @@ if(insource)
MESSAGE(FATAL_ERROR "${PROJECT_NAME} requires an out of source build. Goto ./Build and tapes cmake ../")
endif(insource)
# MPI option has to be set before project, cannot be changed in the cache!
if( SCALFMM_USE_MPI )
include(CMakeForceCompiler)
CMAKE_FORCE_C_COMPILER(mpicc "MPI C Compiler")
CMAKE_FORCE_CXX_COMPILER(mpicxx "MPI C++ Compiler")
else()
MESSAGE(STATUS "Remove CMake cache and run cmake .. -DSCALFMM_USE_MPI=ON to enable MPI" )
endif(SCALFMM_USE_MPI)
# Project Infos
project(ScalFMM)
#===========================================================================
......@@ -19,14 +28,6 @@ SET(ScalFMM_BUILD_VERSION 1)
SET(ScalFMM_VERSION "${ScalFMM_MAJOR_VERSION}.${ScalFMM_MINOR_VERSION}.${ScalFMM_BUILD_VERSION}"
)
# MPI option has to be set before project, cannot be changed in the cache!
if( SCALFMM_USE_MPI )
include(CMakeForceCompiler)
CMAKE_FORCE_C_COMPILER(mpicc "MPI C Compiler")
CMAKE_FORCE_CXX_COMPILER(mpicxx "MPI C++ Compiler")
else()
MESSAGE(STATUS "Remove CMake cache and run cmake .. -DSCALFMM_USE_MPI=ON to enable MPI" )
endif(SCALFMM_USE_MPI)
#
# Active language
# -----------------------
......
......@@ -4,13 +4,13 @@
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
#ifndef FOCTREEARRANGER_HPP
......@@ -32,7 +32,7 @@
* to move the particles in the tree instead of building a new
* tree.
*/
template <class OctreeClass, class ContainerClass, class ExtractorClass>
template <class OctreeClass, class ContainerClass, class ParticleClass, class ConverterClass >
class FOctreeArranger : FAssertable {
OctreeClass* const tree; //< The tree to work on
......@@ -45,7 +45,7 @@ public:
/** Arrange */
void rearrange(const int isPeriodic = DirNone){
// This vector is to keep the moving particles
ExtractorClass extractor;
FVector<ParticleClass> tomove;
// For periodic
const FReal boxWidth = tree->getBoxWidth();
......@@ -128,19 +128,16 @@ public:
const MortonIndex particuleIndex = tree->getMortonFromPosition(partPos);
if(particuleIndex != currentIndex){
indexesToExtract.push(idxPart);
tomove.push(ConverterClass::GetParticle(particles,idxPart));
}
}
// remove from leaf
extractor.extractParticles(particles, indexesToExtract.data(), indexesToExtract.getSize());
particles->removeParticles(indexesToExtract.data(), indexesToExtract.getSize());
indexesToExtract.clear();
} while(octreeIterator.moveRight());
}
{ // insert particles that moved
extractor.reinsertInTree(tree);
for(int idxPart = 0 ; idxPart < tomove.getSize() ; ++idxPart){
ConverterClass::Insert( tree , tomove[idxPart]);
}
}
{ // Remove empty leaves
......
......@@ -33,7 +33,7 @@
* to move the particles in the tree instead of building a new
* tree.
*/
template <class OctreeClass, class ContainerClass, class ParticleClass>
template <class OctreeClass, class ContainerClass, class ParticleClass, class ConverterClass >
class FOctreeArrangerProc : FAssertable {
/** Interval is the min/max morton index
* for a proc
......@@ -105,14 +105,17 @@ public:
const FPoint min(tree->getBoxCenter(),-boxWidth/2);
const FPoint max(tree->getBoxCenter(),boxWidth/2);
FVector<int> indexesToExtract;
typename OctreeClass::Iterator octreeIterator(tree);
octreeIterator.gotoBottomLeft();
do{
const MortonIndex currentIndex = octreeIterator.getCurrentGlobalIndex();
typename ContainerClass::BasicIterator iter(*octreeIterator.getCurrentListTargets());
while( iter.hasNotFinished() ){
FPoint partPos = iter.data().getPosition();
ContainerClass* particles = octreeIterator.getCurrentLeaf()->getSrc();
for(int idxPart = 0 ; idxPart < particles->getNbParticles(); ++idxPart){
FPoint partPos( particles->getPositions()[0][idxPart],
particles->getPositions()[1][idxPart],
particles->getPositions()[2][idxPart] );
// XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
if( TestPeriodicCondition(isPeriodic, DirPlusX) ){
while(partPos.getX() >= max.getX()){
......@@ -171,20 +174,23 @@ public:
printf("Application is exiting...\n");
}
// set pos
iter.data().setPosition(partPos);
particles->getWPositions()[0][idxPart] = partPos.getX();
particles->getWPositions()[1][idxPart] = partPos.getY();
particles->getWPositions()[2][idxPart] = partPos.getZ();
const MortonIndex particuleIndex = tree->getMortonFromPosition(iter.data().getPosition());
const MortonIndex particuleIndex = tree->getMortonFromPosition(partPos);
// is this particle need to be changed from its leaf
if(particuleIndex != currentIndex){
// find the right interval
const int procConcerned = getInterval( particuleIndex, comm.processCount(), intervals);
toMove[procConcerned].push(iter.data());
iter.remove();
}
else {
iter.gotoNext();
toMove[procConcerned].push(ConverterClass::GetParticle(particles,idxPart));
indexesToExtract.push(idxPart);
}
}
particles->removeParticles(indexesToExtract.data(), indexesToExtract.getSize());
indexesToExtract.clear();
} while(octreeIterator.moveRight());
}
......@@ -246,7 +252,7 @@ public:
{ // insert particles that moved
for(int idxPart = 0 ; idxPart < toMove[comm.processId()].getSize() ; ++idxPart){
tree->insert(toMove[comm.processId()][idxPart]);
ConverterClass::Insert( tree , toMove[comm.processId()][idxPart]);
}
}
......@@ -259,7 +265,7 @@ public:
if( done < limitRecvSend ){
const int source = status.MPI_SOURCE;
for(long long int idxPart = indexToReceive[source] ; idxPart < indexToReceive[source+1] ; ++idxPart){
tree->insert(toReceive[idxPart]);
ConverterClass::Insert( tree , toReceive[idxPart]);
}
hasToRecvFrom -= 1;
}
......@@ -274,20 +280,13 @@ public:
do{
// Empty leaf
if( octreeIterator.getCurrentListTargets()->getSize() == 0 ){
if( octreeIterator.getCurrentListTargets()->getNbParticles() == 0 ){
const MortonIndex currentIndex = octreeIterator.getCurrentGlobalIndex();
workOnNext = octreeIterator.moveRight();
tree->removeLeaf( currentIndex );
}
// Not empty, just continue
else {
// todo remove
if( octreeIterator.getCurrentGlobalIndex() < intervals[comm.processId()].min
|| intervals[comm.processId()].max <= octreeIterator.getCurrentGlobalIndex()){
std::cout << comm.processId() << " Must delete this leaf at " << octreeIterator.getCurrentGlobalIndex()
<< " size " << octreeIterator.getCurrentListTargets()->getSize() <<std::endl;
}
workOnNext = octreeIterator.moveRight();
counterLeavesAlive += 1;
}
......
......@@ -27,15 +27,9 @@ class FPoint;
* @brief
* Please read the license
*
* This class define the method that every particle class
* This class define the method that every particle container
* has to implement.
*
* In fact FOctree & FFmmAlgorithm need this function to be implemented.
* But you cannot use this interface with the extension (as an example :
* because the compiler will faill to know if getPosition is coming
* from this interface or from the extension)
*
*
* @warning Inherite from this class when implement a specific particle type
*/
class FAbstractParticleContainer {
......@@ -44,6 +38,11 @@ public:
virtual ~FAbstractParticleContainer(){
}
/**
* This method should be inherited (or your leaf will do nothing)
* the point is coming from the tree and is fallowed by what let the leaf
* pass throught its push method.
*/
template<typename... Args>
void push(const FPoint& /*inParticlePosition*/, Args ... /*args*/){
FDEBUG( FDebug::Controller.write("Warning, push is not implemented!").write(FDebug::Flush) );
......
......@@ -25,112 +25,227 @@
* @class FBasicParticle
* Please read the license
*
* This class defines a basic particle used for examples. It extends
* the mininum, only what is needed by FOctree and FFmmAlgorithm
* to make the things working.
* By using this extension it will implement the FAbstractParticle without
* inheriting from it.
* This class defines a container which can holds one type (AttributeClass)
* for each particle.
* The memory is allocated for all informations, the positions and the
* request type.
* For example is one want to store a struct for each particle:
* @code
* @code struct AStruct{
* @code ...
* @code };
* @code FBasicParticleContainer<1, AStruct> container;
* And then the access is done using:
* @code AStruct* strucs = container.getAttributes<0>();
*/
template <unsigned NbAttributesPerParticle, class AttributeClass = FReal >
class FBasicParticleContainer : public FAbstractParticleContainer {
protected:
/** The number of particles in the container */
int nbParticles;
/** 3 pointers to 3 arrays of real to store the position */
FReal* positions[3];
/** The attributes requested by the user */
AttributeClass* attributes[NbAttributesPerParticle];
/** The allocated memory */
int allocatedParticles;
/////////////////////////////////////////////////////
/////////////////////////////////////////////////////
/** Ending call for pushing the attributes */
template<int index>
void addParticleValue(const int /*insertPosition*/){
}
/** Filling call for each attributes values */
template<int index, typename... Args>
void addParticleValue(const int insertPosition, const AttributeClass value, Args... args){
// Compile test to ensure indexing
static_assert(index < NbAttributesPerParticle, "Index to get attributes is out of scope.");
// insert the value
attributes[index][insertPosition] = value;
// Continue for reamining values
addParticleValue<index+1>( insertPosition, args...);
}
public:
/////////////////////////////////////////////////////
/////////////////////////////////////////////////////
FBasicParticleContainer(const FBasicParticleContainer&) = delete;
FBasicParticleContainer& operator=(const FBasicParticleContainer&) = delete;
/////////////////////////////////////////////////////
/////////////////////////////////////////////////////
/** Basic contructor */
FBasicParticleContainer() : nbParticles(0), allocatedParticles(0){
memset(positions, 0, sizeof(positions[0]) * 3);
memset(attributes, 0, sizeof(attributes[0]) * NbAttributesPerParticle);
}
/** Simply dalloc the memory using first pointer
*/
~FBasicParticleContainer(){
delete[] reinterpret_cast<char*>(positions[0]);
}
/**
* @brief getNbParticles
* @return the number of particles
*/
int getNbParticles() const{
return nbParticles;
}
/**
* @brief getPositions
* @return a FReal*[3] to get access to the positions
*/
const FReal*const*const getPositions() const {
return positions;
}
/**
* @brief getWPositions
* @return get the position in write mode
*/
FReal*const*const getWPositions() {
return positions;
}
/**
* @brief getAttribute
* @param index
* @return the attribute at index index
*/
AttributeClass* getAttribute(const int index) {
return attributes[index];
}
/**
* @brief getAttribute
* @param index
* @return
*/
const AttributeClass* getAttribute(const int index) const {
return attributes[index];
}
/**
* Get the attribute with a forcing compile optimization
*/
template <int index>
AttributeClass* getAttribute() {
static_assert(index < NbAttributesPerParticle, "Index to get attributes is out of scope.");
return attributes[index];
}
/**
* Get the attribute with a forcing compile optimization
*/
template <int index>
const AttributeClass* getAttribute() const {
static_assert(index < NbAttributesPerParticle, "Index to get attributes is out of scope.");
return attributes[index];
}
/////////////////////////////////////////////////////
/////////////////////////////////////////////////////
/**
* Push called bu FSimpleLeaf
* Should have a particle position fallowed by attributes
*/
template<typename... Args>
void push(const FPoint& inParticlePosition, Args... args){
// enought space?
if( nbParticles == allocatedParticles ){
// allocate memory
allocatedParticles = FMath::Max(10,int(FReal(nbParticles+1)*1.5));
// init with 0
FReal* newData = reinterpret_cast<FReal*>(new char[(sizeof(FReal)*3 + sizeof(AttributeClass)*NbAttributesPerParticle)*allocatedParticles]);
memset( newData, 0, (sizeof(FReal)*3 + sizeof(AttributeClass)*NbAttributesPerParticle)*allocatedParticles);
// copy memory
const char*const toDelete = reinterpret_cast<const char*>(positions[0]);
for(int idx = 0 ; idx < 3 ; ++idx){
memcpy(newData + (allocatedParticles * idx), positions[idx], sizeof(FReal) * nbParticles);
positions[idx] = newData + (allocatedParticles * idx);
}
// copy attributes
AttributeClass* startAddress = reinterpret_cast<AttributeClass*>(positions[2] + allocatedParticles);
for(unsigned idx = 0 ; idx < NbAttributesPerParticle ; ++idx){
memcpy(startAddress + (allocatedParticles * idx), attributes[idx], sizeof(AttributeClass) * nbParticles);
attributes[idx] = startAddress + (idx * allocatedParticles);
}
// delete old
delete[] toDelete;
}
// insert particle data
positions[0][nbParticles] = inParticlePosition.getX();
positions[1][nbParticles] = inParticlePosition.getY();
positions[2][nbParticles] = inParticlePosition.getZ();
// insert attribute data
addParticleValue<0>( nbParticles, args...);
nbParticles += 1;
}
/**
* Push called usually by FTypedLeaf with the isTarget flag in addition
*/
template<typename... Args>
void push(const FPoint& inParticlePosition, const bool /*isTarget*/, Args... args){
push(inParticlePosition, args...);
}
/** set nb particles to 0 */
void clear(){
nbParticles = 0;
}
/** to enable rearranging
* indexesToRemove must be sorted
* it removes all the particles at position indexesToRemove
*/
void removeParticles(const int indexesToRemove[], const int nbParticlesToRemove){
int offset = 1;
int idxIndexes = 1;
int idxIns = indexesToRemove[0] + 1;
for( ; idxIns < nbParticles && idxIndexes < nbParticlesToRemove ; ++idxIns){
if( idxIns == indexesToRemove[idxIndexes] ){
idxIndexes += 1;
offset += 1;
}
else{
for(int idx = 0 ; idx < 3 ; ++idx){
positions[idx][idxIns-offset] = positions[idx][idxIns];
}
for(unsigned idx = 0 ; idx < NbAttributesPerParticle ; ++idx){
attributes[idx][idxIns-offset] = attributes[idx][idxIns];
}
}
}
for( ; idxIns < nbParticles ; ++idxIns){
for(int idx = 0 ; idx < 3 ; ++idx){
positions[idx][idxIns-offset] = positions[idx][idxIns];
}
for(unsigned idx = 0 ; idx < NbAttributesPerParticle ; ++idx){
attributes[idx][idxIns-offset] = attributes[idx][idxIns];
}
}
nbParticles -= nbParticlesToRemove;
}
/////////////////////////////////////////////////////
/////////////////////////////////////////////////////
/** The size to send a leaf */
int getSavedSize() const{
return int(nbParticles * (3 * sizeof(FReal) + NbAttributesPerParticle * sizeof(AttributeClass)));
}
/** Save the current cell in a buffer */
void save(FBufferWriter& buffer) const{
buffer << nbParticles;
......@@ -166,36 +281,6 @@ public:
}
}
/** to enable rearranging */
void removeParticles(const int indexesToRemove[], const int nbParticlesToRemove){
int offset = 1;
int idxIndexes = 1;
int idxIns = indexesToRemove[0] + 1;
for( ; idxIns < nbParticles && idxIndexes < nbParticlesToRemove ; ++idxIns){
if( idxIns == indexesToRemove[idxIndexes] ){
idxIndexes += 1;
offset += 1;
}
else{
for(int idx = 0 ; idx < 3 ; ++idx){
positions[idx][idxIns-offset] = positions[idx][idxIns];
}
for(unsigned idx = 0 ; idx < NbAttributesPerParticle ; ++idx){
attributes[idx][idxIns-offset] = attributes[idx][idxIns];
}
}
}
for( ; idxIns < nbParticles ; ++idxIns){
for(int idx = 0 ; idx < 3 ; ++idx){
positions[idx][idxIns-offset] = positions[idx][idxIns];
}
for(unsigned idx = 0 ; idx < NbAttributesPerParticle ; ++idx){
attributes[idx][idxIns-offset] = attributes[idx][idxIns];
}
}
nbParticles -= nbParticlesToRemove;
}
};
......
......@@ -23,19 +23,27 @@
* @class FFmaParticle
* Please read the license
*
* This class defines a particle for FMA loader.
* As defined in FFmaLoader it needs
* This class defines a particle container for FMA loader.
* position + 1 real for the physical value
*/
class FFmaParticleContainer : public FBasicParticleContainer<1> {
typedef FBasicParticleContainer<1> Parent;
public:
/**
* @brief getPhysicalValues to get the array of physical values
* @return
*/
FReal* getPhysicalValues(){
return Parent::getAttribute(0);
return Parent::getAttribute<0>();
}
/**
* @brief getPhysicalValues to get the array of physical values
* @return
*/
const FReal* getPhysicalValues() const {
return Parent::getAttribute(0);
return Parent::getAttribute<0>();
}
};
......
......@@ -37,30 +37,31 @@ public:
}
/**
* To add a new particle in the leaf
* @param particle the new particle to store in the current leaf
*/
* To add a new particle in the leaf
* @param inParticlePosition the new particle position to store in the current leaf
* and the other parameters given by the user
*/
template<typename... Args>
void push(const FPoint& inParticlePosition, Args ... args){
// We pass every thing to the container and let it manage
this->particles.push(inParticlePosition, args...);
}
/**
* To get all the sources in a leaf
* @return a pointer to the list of particles that are sources
*/
* To get all the sources in a leaf
* @return a pointer to the list of particles that are sources
*/
ContainerClass* getSrc() {
return &this->particles;
}
/**
* To get all the target in a leaf
* @return a pointer to the list of particles that are targets
*/
* To get all the target in a leaf
* @return a pointer to the list of particles that are targets
*/
ContainerClass* getTargets() {
return &this->particles;
}
};
......
......@@ -26,20 +26,26 @@
*
* This class is used in the FTestKernels, please
* look at this class to know whit it is.
*
* Particles just need the data down (after and run with FTestKernel the
* value shoud be NB PARTICLES (-1)).
* We store the positions + 1 long long int
*/
class FTestParticleContainer : public FBasicParticleContainer<1, long long int> {
typedef FBasicParticleContainer<1, long long int> Parent;
public:
/**
* @brief getDataDown
* @return
*/
long long int* getDataDown(){
return Parent::getAttribute(0);
return Parent::getAttribute<0>();
}
/**
* @brief getDataDown
* @return
*/
const long long int* getDataDown() const {
return Parent::getAttribute(0);
return Parent::getAttribute<0>();
}
};
......
......@@ -42,26 +42,28 @@ public:
/**
* To add a new particle in the leaf
* @param particle the new particle
* @param inParticlePosition the position of the new particle
* @param isTarget bool to know if it is a target
* followed by other param given by the user
*/
template<typename... Args>
void push(const FPoint& inParticlePosition, const bool isTarget, Args ... args){
if(isTarget) targets.push(inParticlePosition, args...);
else sources.push(inParticlePosition, args...);
if(isTarget) targets.push(inParticlePosition, isTarget, args...);
else sources.push(inParticlePosition, isTarget, args...);
}
/**
* To get all the sources in a leaf
* @return a pointer to the list of particles that are sources
*/
* To get all the sources in a leaf
* @return a pointer to the list of particles that are sources
*/
ContainerClass* getSrc() {
return &this->sources;
}
/**
* To get all the target in a leaf
* @return a pointer to the list of particles that are targets
*/
* To get all the target in a leaf
* @return a pointer to the list of particles that are targets
*/
ContainerClass* getTargets() {
return &this->targets;
}
......
......@@ -1018,7 +1018,8 @@ private:
alreadySent[procToReceive] = 1;
toSend[procToReceive].push( iterArray[idxLeaf] );
partsToSend[procToReceive] += iterArray[idxLeaf].getCurrentListSrc()->getSize();
partsToSend[procToReceive] += iterArray[idxLeaf].getCurrentListSrc()->getSavedSize();
partsToSend[procToReceive] += int(sizeof(MortonIndex));
}
}
}
......@@ -1029,6 +1030,12 @@ private:
}
}
for(int idxProc = 0 ; idxProc < nbProcess ; ++idxProc){
if(partsToSend[idxProc]){
partsToSend[idxProc] += int(sizeof(int));
}
}
FDEBUG(gatherCounter.tic());
FMpi::MpiAssert( MPI_Allgather( partsToSend, nbProcess, MPI_INT, globalReceiveMap, nbProcess, MPI_INT, comm.getComm()), __LINE__ );
FDEBUG(gatherCounter.tac());
......@@ -1036,7 +1043,7 @@ private:
// Prepare receive
for(int idxProc = 0 ; idxProc < nbProcess ; ++idxProc){
if(globalReceiveMap[idxProc * nbProcess + idProcess]){
recvBuffer[idxProc] = new FBufferReader(int(sizeof(ParticleClass)) * globalReceiveMap[idxProc * nbProcess + idProcess]);
recvBuffer[idxProc] = new FBufferReader(globalReceiveMap[idxProc * nbProcess + idProcess]);
FMpi::MpiAssert( MPI_Irecv(recvBuffer[idxProc]->data(), recvBuffer[idxProc]->getSize(), MPI_BYTE,
idxProc, FMpi::TagFmmP2P, comm.getComm(), &requests[iterRequest++]) , __LINE__ );
}
......@@ -1046,16 +1053,20 @@ private:
// Prepare send
for(int idxProc = 0 ; idxProc < nbProcess ; ++idxProc){
if(toSend[idxProc].getSize() != 0){
sendBuffer[idxProc] = new FBufferWriter(int(sizeof(ParticleClass)) * partsToSend[idxProc]);
sendBuffer[idxProc] = new FBufferWriter(partsToSend[idxProc]);
(*sendBuffer[idxProc]) << toSend[idxProc].getSize();
for(int idxLeaf = 0 ; idxLeaf < toSend[idxProc].getSize() ; ++idxLeaf){
typename ContainerClass::BasicIterator iterSource(*toSend[idxProc][idxLeaf].getCurrentListSrc());
while( iterSource.hasNotFinished() ){
iterSource.data().save(*sendBuffer[idxProc]);
iterSource.gotoNext();
}
(*sendBuffer[idxProc]) << toSend[idxProc][idxLeaf].getCurrentGlobalIndex();
toSend[idxProc][idxLeaf].getCurrentListSrc()->save(*sendBuffer[idxProc]);
}
#ifdef FUSE_DEBUG
// TODO clean test
if(sendBuffer[idxProc]->getSize() != partsToSend[idxProc]){
printf("Error 1056 fmm algo proc\n");
}
#endif
FMpi::MpiAssert( MPI_Isend( sendBuffer[idxProc]->data(), sendBuffer[idxProc]->getSize() , MPI_BYTE ,
idxProc, FMpi::TagFmmP2P, comm.getComm(), &requests[iterRequest++]) , __LINE__ );
......@@ -1185,10 +1196,12 @@ private:
for(int idxRcv = 0 ; idxRcv < countMessages ; ++idxRcv){
if( indexMessage[idxRcv] < nbMessagesToRecv ){
const int idxProc = status[idxRcv].MPI_SOURCE;
ParticleClass tempoParticle;
for(int idxPart = 0 ; idxPart < globalReceiveMap[idxProc * nbProcess + idProcess] ; ++idxPart){
tempoParticle.restore(*recvBuffer[idxProc]);
otherP2Ptree.insert(tempoParticle);
int nbLeaves;
(*recvBuffer[idxProc]) >> nbLeaves;
for(int idxLeaf = 0 ; idxLeaf < nbLeaves ; ++idxLeaf){
MortonIndex leafIndex;
(*recvBuffer[idxProc]) >> leafIndex;
otherP2Ptree.createLeaf(leafIndex)->getSrc()->restore((*recvBuffer[idxProc]));
}
delete recvBuffer[idxProc];