Commit c5cc6b92 authored by PIACIBELLO Cyrille's avatar PIACIBELLO Cyrille
Browse files

FMpiTreeBuilder has been modified in order to work with a call from >>mpirun -np 1 ...

parent 3edeb822
...@@ -149,27 +149,30 @@ private: ...@@ -149,27 +149,30 @@ private:
FTRACE( FTrace::FFunction functionTrace(__FUNCTION__, "Loader to Tree" , __FILE__ , __LINE__) ); FTRACE( FTrace::FFunction functionTrace(__FUNCTION__, "Loader to Tree" , __FILE__ , __LINE__) );
const int rank = communicator.processId(); const int rank = communicator.processId();
const int nbProcs = communicator.processCount(); const int nbProcs = communicator.processCount();
if(nbProcs == 1){
// be sure there is no splited leaves //Nothing to do there : there is no need to verify if leaves are split, if there is one process...
// to do that we exchange the first index with the left proc }
{ else{
// be sure there is no splited leaves
// to do that we exchange the first index with the left proc
{
FTRACE( FTrace::FRegion regionTrace("Remove Splited leaves", __FUNCTION__ , __FILE__ , __LINE__) ); FTRACE( FTrace::FRegion regionTrace("Remove Splited leaves", __FUNCTION__ , __FILE__ , __LINE__) );
MortonIndex otherFirstIndex = -1; MortonIndex otherFirstIndex = -1;
if(workingSize != 0 && rank != 0 && rank != nbProcs - 1){ if(workingSize != 0 && rank != 0 && rank != nbProcs - 1){
MPI_Sendrecv(&workingArray[0].index, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs, MPI_Sendrecv(&workingArray[0].index, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs,
&otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs, &otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs,
communicator.getComm(), MPI_STATUS_IGNORE); communicator.getComm(), MPI_STATUS_IGNORE);
} }
else if( rank == 0){ else if( rank == 0){
MPI_Recv(&otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs, communicator.getComm(), MPI_STATUS_IGNORE); MPI_Recv(&otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs, communicator.getComm(), MPI_STATUS_IGNORE);
} }
else if( rank == nbProcs - 1){ else if( rank == nbProcs - 1){
MPI_Send( &workingArray[0].index, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs, communicator.getComm()); MPI_Send( &workingArray[0].index, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs, communicator.getComm());
} }
else { else {
MPI_Recv(&otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs, communicator.getComm(), MPI_STATUS_IGNORE); MPI_Recv(&otherFirstIndex, 1, MPI_LONG_LONG, rank + 1, FMpi::TagExchangeIndexs, communicator.getComm(), MPI_STATUS_IGNORE);
MPI_Send(&otherFirstIndex, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs, communicator.getComm()); MPI_Send(&otherFirstIndex, 1, MPI_LONG_LONG, rank - 1, FMpi::TagExchangeIndexs, communicator.getComm());
} }
// at this point every one know the first index of his right neighbors // at this point every one know the first index of his right neighbors
...@@ -178,103 +181,104 @@ private: ...@@ -178,103 +181,104 @@ private:
IndexedParticle* sendBuffer = 0; IndexedParticle* sendBuffer = 0;
if(rank != nbProcs - 1 && needToRecvBeforeSend == false){ if(rank != nbProcs - 1 && needToRecvBeforeSend == false){
FSize idxPart = workingSize - 1 ; FSize idxPart = workingSize - 1 ;
while(idxPart >= 0 && workingArray[idxPart].index == otherFirstIndex){ while(idxPart >= 0 && workingArray[idxPart].index == otherFirstIndex){
--idxPart; --idxPart;
} }
const int particlesToSend = int(workingSize - 1 - idxPart); const int particlesToSend = int(workingSize - 1 - idxPart);
if(particlesToSend){ if(particlesToSend){
workingSize -= particlesToSend; workingSize -= particlesToSend;
sendBuffer = new IndexedParticle[particlesToSend]; sendBuffer = new IndexedParticle[particlesToSend];
memcpy(sendBuffer, &workingArray[idxPart + 1], particlesToSend * sizeof(IndexedParticle)); memcpy(sendBuffer, &workingArray[idxPart + 1], particlesToSend * sizeof(IndexedParticle));
MPI_Isend( sendBuffer, particlesToSend * int(sizeof(IndexedParticle)), MPI_BYTE, MPI_Isend( sendBuffer, particlesToSend * int(sizeof(IndexedParticle)), MPI_BYTE,
rank + 1, FMpi::TagSplittedLeaf, communicator.getComm(), &requestSendLeaf); rank + 1, FMpi::TagSplittedLeaf, communicator.getComm(), &requestSendLeaf);
} }
else{ else{
MPI_Isend( 0, 0, MPI_BYTE, rank + 1, FMpi::TagSplittedLeaf, communicator.getComm(), &requestSendLeaf); MPI_Isend( 0, 0, MPI_BYTE, rank + 1, FMpi::TagSplittedLeaf, communicator.getComm(), &requestSendLeaf);
} }
} }
if( rank != 0 ){ if( rank != 0 ){
int sendByOther = 0; int sendByOther = 0;
MPI_Status probStatus; MPI_Status probStatus;
MPI_Probe(rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), &probStatus); MPI_Probe(rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), &probStatus);
MPI_Get_count( &probStatus, MPI_BYTE, &sendByOther); MPI_Get_count( &probStatus, MPI_BYTE, &sendByOther);
if(sendByOther){ if(sendByOther){
sendByOther /= int(sizeof(IndexedParticle)); sendByOther /= int(sizeof(IndexedParticle));
const IndexedParticle* const reallocOutputArray = workingArray; const IndexedParticle* const reallocOutputArray = workingArray;
const FSize reallocOutputSize = workingSize; const FSize reallocOutputSize = workingSize;
workingSize += sendByOther; workingSize += sendByOther;
workingArray = new IndexedParticle[workingSize]; workingArray = new IndexedParticle[workingSize];
FMemUtils::memcpy(&workingArray[sendByOther], reallocOutputArray, reallocOutputSize * sizeof(IndexedParticle)); FMemUtils::memcpy(&workingArray[sendByOther], reallocOutputArray, reallocOutputSize * sizeof(IndexedParticle));
delete[] reallocOutputArray; delete[] reallocOutputArray;
MPI_Recv(workingArray, int(sizeof(IndexedParticle)) * sendByOther, MPI_BYTE, MPI_Recv(workingArray, int(sizeof(IndexedParticle)) * sendByOther, MPI_BYTE,
rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), MPI_STATUS_IGNORE); rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), MPI_STATUS_IGNORE);
} }
else{ else{
MPI_Recv( 0, 0, MPI_BYTE, rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), MPI_STATUS_IGNORE); MPI_Recv( 0, 0, MPI_BYTE, rank - 1, FMpi::TagSplittedLeaf, communicator.getComm(), MPI_STATUS_IGNORE);
} }
} }
if(rank != nbProcs - 1 && needToRecvBeforeSend == true){ if(rank != nbProcs - 1 && needToRecvBeforeSend == true){
MPI_Send( workingArray, int(workingSize * sizeof(IndexedParticle)), MPI_BYTE, MPI_Send( workingArray, int(workingSize * sizeof(IndexedParticle)), MPI_BYTE,
rank + 1, FMpi::TagSplittedLeaf, communicator.getComm()); rank + 1, FMpi::TagSplittedLeaf, communicator.getComm());
delete[] workingArray; delete[] workingArray;
workingArray = 0; workingArray = 0;
workingSize = 0; workingSize = 0;
} }
else if(rank != nbProcs - 1){ else if(rank != nbProcs - 1){
MPI_Wait( &requestSendLeaf, MPI_STATUS_IGNORE); MPI_Wait( &requestSendLeaf, MPI_STATUS_IGNORE);
delete[] sendBuffer; delete[] sendBuffer;
sendBuffer = 0; sendBuffer = 0;
} }
} }
}
{ {
FTRACE( FTrace::FRegion regionTrace("Remove Splited leaves", __FUNCTION__ , __FILE__ , __LINE__) ); FTRACE( FTrace::FRegion regionTrace("Remove Splited leaves", __FUNCTION__ , __FILE__ , __LINE__) );
// We now copy the data from a sorted type into real particles array + counter // We now copy the data from a sorted type into real particles array + counter
(*leavesSize) = 0; (*leavesSize) = 0;
(*leavesArray) = 0; (*leavesArray) = 0;
if(workingSize){ if(workingSize){
(*leavesArray) = new char[workingSize * (sizeof(ParticleClass) + sizeof(int))]; (*leavesArray) = new char[workingSize * (sizeof(ParticleClass) + sizeof(int))];
MortonIndex previousIndex = -1; MortonIndex previousIndex = -1;
char* writeIndex = (*leavesArray); char* writeIndex = (*leavesArray);
int* writeCounter = 0; int* writeCounter = 0;
for( FSize idxPart = 0; idxPart < workingSize ; ++idxPart){ for( FSize idxPart = 0; idxPart < workingSize ; ++idxPart){
if( workingArray[idxPart].index != previousIndex ){ if( workingArray[idxPart].index != previousIndex ){
previousIndex = workingArray[idxPart].index; previousIndex = workingArray[idxPart].index;
++(*leavesSize); ++(*leavesSize);
writeCounter = reinterpret_cast<int*>( writeIndex ); writeCounter = reinterpret_cast<int*>( writeIndex );
writeIndex += sizeof(int); writeIndex += sizeof(int);
(*writeCounter) = 0; (*writeCounter) = 0;
} }
memcpy(writeIndex, &workingArray[idxPart].particle, sizeof(ParticleClass)); memcpy(writeIndex, &workingArray[idxPart].particle, sizeof(ParticleClass));
writeIndex += sizeof(ParticleClass); writeIndex += sizeof(ParticleClass);
++(*writeCounter); ++(*writeCounter);
} }
} }
delete [] workingArray; delete [] workingArray;
workingArray = 0; workingArray = 0;
workingSize = 0; workingSize = 0;
} }
} }
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
// To equalize (same number of leaves among the procs) // To equalize (same number of leaves among the procs)
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
...@@ -289,232 +293,251 @@ private: ...@@ -289,232 +293,251 @@ private:
const int myRank = communicator.processId(); const int myRank = communicator.processId();
const int nbProcs = communicator.processCount(); const int nbProcs = communicator.processCount();
const FSize myNumberOfLeaves = nbLeavesInIntervals; const FSize myNumberOfLeaves = nbLeavesInIntervals;
//Test if -np 1
// We have to know the number of leaves each procs holds if(nbProcs == 1){
int*const numberOfLeavesPerProc = new int[nbProcs]; //there is no need to equalize tree between if there is one process
memset(numberOfLeavesPerProc, 0, sizeof(int) * nbProcs); //Saving directly the particles
int intNbLeavesInIntervals = int(nbLeavesInIntervals); int counter = 0;
MPI_Allgather(&intNbLeavesInIntervals, 1, MPI_INT, numberOfLeavesPerProc, 1, MPI_INT, communicator.getComm()); FSize idxInLeafArray = 0;
printf("Proc : %d : Currently have %lld leaves \n", myRank, myNumberOfLeaves); for(FSize idxLeaf = 0 ; idxLeaf < nbLeavesInIntervals ; idxLeaf++){
counter++;
//Start working THERE int nbPartInLeaf = *(reinterpret_cast<int*>(&leavesArray[idxInLeafArray]));
idxInLeafArray += sizeof(int);
//We need the max number of leafs over th procs for(FSize idxPartInLeaf = 0 ; idxPartInLeaf<nbPartInLeaf ; idxPartInLeaf++){
int*const leavesOffsetPerProc = new int[nbProcs + 1]; particlesSaver->push(*(reinterpret_cast<ParticleClass*>(&leavesArray[idxInLeafArray+idxPartInLeaf*sizeof(ParticleClass)])));
FSize totalNumberOfLeaves = 0; }
idxInLeafArray += sizeof(ParticleClass)*nbPartInLeaf;
leavesOffsetPerProc[0] = 0; }
totalNumberOfLeaves += numberOfLeavesPerProc[0]; printf("nbLeavesInIntervals %d \n",nbLeavesInIntervals);
for(int idxProc = 1 ; idxProc < nbProcs ; ++idxProc ){ }
else{
// We have to know the number of leaves each procs holds
int*const numberOfLeavesPerProc = new int[nbProcs];
memset(numberOfLeavesPerProc, 0, sizeof(int) * nbProcs);
int intNbLeavesInIntervals = int(nbLeavesInIntervals);
MPI_Allgather(&intNbLeavesInIntervals, 1, MPI_INT, numberOfLeavesPerProc, 1, MPI_INT, communicator.getComm());
printf("Proc : %d : Currently have %lld leaves \n", myRank, myNumberOfLeaves);
//Start working THERE
//We need the max number of leafs over th procs
int*const leavesOffsetPerProc = new int[nbProcs + 1];
FSize totalNumberOfLeaves = 0;
leavesOffsetPerProc[0] = 0;
totalNumberOfLeaves += numberOfLeavesPerProc[0];
for(int idxProc = 1 ; idxProc < nbProcs ; ++idxProc ){
leavesOffsetPerProc[idxProc] = leavesOffsetPerProc[idxProc-1] + numberOfLeavesPerProc[idxProc-1]; leavesOffsetPerProc[idxProc] = leavesOffsetPerProc[idxProc-1] + numberOfLeavesPerProc[idxProc-1];
totalNumberOfLeaves += numberOfLeavesPerProc[idxProc]; totalNumberOfLeaves += numberOfLeavesPerProc[idxProc];
} }
leavesOffsetPerProc[nbProcs] = int(totalNumberOfLeaves); leavesOffsetPerProc[nbProcs] = int(totalNumberOfLeaves);
const FSize currentLeafsOnMyLeft = leavesOffsetPerProc[myRank]; const FSize currentLeafsOnMyLeft = leavesOffsetPerProc[myRank];
const FSize currentRightLeafIdx = leavesOffsetPerProc[myRank+1]; const FSize currentRightLeafIdx = leavesOffsetPerProc[myRank+1];
//Creation of an array to store how many parts are in each leaf //Creation of an array to store how many parts are in each leaf
int*const numberOfParticlesPerLeaf = new int[totalNumberOfLeaves]; int*const numberOfParticlesPerLeaf = new int[totalNumberOfLeaves];
int*const myParticlesCounterArray = &numberOfParticlesPerLeaf[leavesOffsetPerProc[myRank]]; int*const myParticlesCounterArray = &numberOfParticlesPerLeaf[leavesOffsetPerProc[myRank]];
memset(numberOfParticlesPerLeaf, 0, sizeof(int)*totalNumberOfLeaves); memset(numberOfParticlesPerLeaf, 0, sizeof(int)*totalNumberOfLeaves);
//Loop over leafArray to fill myParts //Loop over leafArray to fill myParts
size_t idxOfParticlesNumber = 0; size_t idxOfParticlesNumber = 0;
for(int idxLeaf = 0 ; idxLeaf < nbLeavesInIntervals ; ++idxLeaf){ for(int idxLeaf = 0 ; idxLeaf < nbLeavesInIntervals ; ++idxLeaf){
const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[idxOfParticlesNumber])); const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[idxOfParticlesNumber]));
myParticlesCounterArray[idxLeaf] += numberOfParticlesInThisLeaf; myParticlesCounterArray[idxLeaf] += numberOfParticlesInThisLeaf;
idxOfParticlesNumber += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int)); idxOfParticlesNumber += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int));
} }
MPI_Allgatherv(myParticlesCounterArray,numberOfLeavesPerProc[myRank], MPI_INT, MPI_Allgatherv(myParticlesCounterArray,numberOfLeavesPerProc[myRank], MPI_INT,
numberOfParticlesPerLeaf, numberOfLeavesPerProc, leavesOffsetPerProc, MPI_INT, communicator.getComm()); numberOfParticlesPerLeaf, numberOfLeavesPerProc, leavesOffsetPerProc, MPI_INT, communicator.getComm());
FSize totalNumberOfParticles = 0; FSize totalNumberOfParticles = 0;
for(int idxLeaf = 0 ; idxLeaf < totalNumberOfLeaves ; ++idxLeaf){ for(int idxLeaf = 0 ; idxLeaf < totalNumberOfLeaves ; ++idxLeaf){
totalNumberOfParticles += numberOfParticlesPerLeaf[idxLeaf]; totalNumberOfParticles += numberOfParticlesPerLeaf[idxLeaf];
} }
const FSize correctLeftLeavesNumber = balancer->getLeft( totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles, const FSize correctLeftLeavesNumber = balancer->getLeft( totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,
NULL,nbProcs,myRank); NULL,nbProcs,myRank);
const FSize correctRightLeavesIndex = balancer->getRight(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles, const FSize correctRightLeavesIndex = balancer->getRight(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,
NULL,nbProcs,myRank); NULL,nbProcs,myRank);
//// TODO REMOVE WHEN DEBUG printf("Proc [%d] :: will work from leaf %lld \t to leaf %lld \n",myRank,correctLeftLeavesNumber,correctRightLeavesIndex); //// TODO REMOVE WHEN DEBUG printf("Proc [%d] :: will work from leaf %lld \t to leaf %lld \n",myRank,correctLeftLeavesNumber,correctRightLeavesIndex);
MPI_Request* requests = new MPI_Request[nbProcs * 2]; MPI_Request* requests = new MPI_Request[nbProcs * 2];
int counterRequest = 0; int counterRequest = 0;
if(currentLeafsOnMyLeft < correctLeftLeavesNumber || correctRightLeavesIndex < currentRightLeafIdx){ if(currentLeafsOnMyLeft < correctLeftLeavesNumber || correctRightLeavesIndex < currentRightLeafIdx){
size_t offsetLeafToSend = 0; size_t offsetLeafToSend = 0;
int counterLeafToSend = 0; int counterLeafToSend = 0;
int idxProcToProceed = 0; int idxProcToProceed = 0;
while( idxProcToProceed < nbProcs && (balancer->getLeft(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed) < currentRightLeafIdx)){ while( idxProcToProceed < nbProcs && (balancer->getLeft(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed) < currentRightLeafIdx)){
const FSize procToProceedRightIdx = balancer->getRight(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed); const FSize procToProceedRightIdx = balancer->getRight(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed);
const FSize procToProceedLeftIdx = balancer->getLeft(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed); const FSize procToProceedLeftIdx = balancer->getLeft(totalNumberOfLeaves,numberOfParticlesPerLeaf,totalNumberOfParticles,NULL,nbProcs,idxProcToProceed);
const bool procToProceedHasLeftInMyInterval = (currentLeafsOnMyLeft <= procToProceedLeftIdx && procToProceedLeftIdx < currentRightLeafIdx); const bool procToProceedHasLeftInMyInterval = (currentLeafsOnMyLeft <= procToProceedLeftIdx && procToProceedLeftIdx < currentRightLeafIdx);
const bool procToProceedHasRightInMyInterval = (currentLeafsOnMyLeft <= procToProceedRightIdx && procToProceedRightIdx < currentRightLeafIdx); const bool procToProceedHasRightInMyInterval = (currentLeafsOnMyLeft <= procToProceedRightIdx && procToProceedRightIdx < currentRightLeafIdx);
const bool procIncludeMyInterval = (procToProceedLeftIdx <= currentLeafsOnMyLeft && currentRightLeafIdx <= procToProceedRightIdx); const bool procIncludeMyInterval = (procToProceedLeftIdx <= currentLeafsOnMyLeft && currentRightLeafIdx <= procToProceedRightIdx);
//// TODO REMOVE WHEN DEBUG printf("%d] idxProcToProceed %d procToProceedRightIdx %llu procToProceedLeftIdx %llu procToProceedHasLeftInMyInterval %d procToProceedHasRightInMyInterval %d\n", //// TODO REMOVE WHEN DEBUG printf("%d] idxProcToProceed %d procToProceedRightIdx %llu procToProceedLeftIdx %llu procToProceedHasLeftInMyInterval %d procToProceedHasRightInMyInterval %d\n",
//// TODO REMOVE WHEN DEBUG myRank, idxProcToProceed, procToProceedRightIdx, procToProceedLeftIdx, procToProceedHasLeftInMyInterval, procToProceedHasRightInMyInterval); //// TODO REMOVE WHEN DEBUG myRank, idxProcToProceed, procToProceedRightIdx, procToProceedLeftIdx, procToProceedHasLeftInMyInterval, procToProceedHasRightInMyInterval);
if(idxProcToProceed != myRank && (procToProceedHasLeftInMyInterval || procToProceedHasRightInMyInterval || procIncludeMyInterval) ){ if(idxProcToProceed != myRank && (procToProceedHasLeftInMyInterval || procToProceedHasRightInMyInterval || procIncludeMyInterval) ){
const int firstLeafToSend = FMath::Max(int(procToProceedLeftIdx - currentLeafsOnMyLeft), 0); const int firstLeafToSend = FMath::Max(int(procToProceedLeftIdx - currentLeafsOnMyLeft), 0);
const int lastLeafToSend = int(FMath::Min(procToProceedRightIdx - currentLeafsOnMyLeft, myNumberOfLeaves )); const int lastLeafToSend = int(FMath::Min(procToProceedRightIdx - currentLeafsOnMyLeft, myNumberOfLeaves ));
//// TODO REMOVE WHEN DEBUG printf("Proc :: %d (from leaf %d to %d)\n", myRank, firstLeafToSend, lastLeafToSend); //// TODO REMOVE WHEN DEBUG printf("Proc :: %d (from leaf %d to %d)\n", myRank, firstLeafToSend, lastLeafToSend);
while(counterLeafToSend != firstLeafToSend){ while(counterLeafToSend != firstLeafToSend){
const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[offsetLeafToSend])); const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[offsetLeafToSend]));
offsetLeafToSend += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int)); offsetLeafToSend += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int));
counterLeafToSend += 1; counterLeafToSend += 1;
} }
const size_t offetSetToSend = offsetLeafToSend; const size_t offetSetToSend = offsetLeafToSend;
while(counterLeafToSend != lastLeafToSend){ while(counterLeafToSend != lastLeafToSend){
const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[offsetLeafToSend])); const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&leavesArray[offsetLeafToSend]));
offsetLeafToSend += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int)); offsetLeafToSend += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int));
counterLeafToSend += 1; counterLeafToSend += 1;
} }
//// TODO REMOVE WHEN DEBUG printf("Proc :: %d send %d bytes to %d (from leaf %d to %d)\n", //// TODO REMOVE WHEN DEBUG printf("Proc :: %d send %d bytes to %d (from leaf %d to %d)\n",
//// TODO REMOVE WHEN DEBUG myRank, int(offsetLeafToSend - offetSetToSend), idxProcToProceed, firstLeafToSend, lastLeafToSend); //// TODO REMOVE WHEN DEBUG myRank, int(offsetLeafToSend - offetSetToSend), idxProcToProceed, firstLeafToSend, lastLeafToSend);
MPI_Isend(&leavesArray[offetSetToSend], int(offsetLeafToSend - offetSetToSend), MPI_BYTE, MPI_Isend(&leavesArray[offetSetToSend], int(offsetLeafToSend - offetSetToSend), MPI_BYTE,
idxProcToProceed, firstLeafToSend + int(currentLeafsOnMyLeft), communicator.getComm(), &requests[counterRequest++]); idxProcToProceed, firstLeafToSend + int(currentLeafsOnMyLeft), communicator.getComm(), &requests[counterRequest++]);
} }
idxProcToProceed += 1; idxProcToProceed += 1;
} }
} }
struct RecvBlockInfo{ struct RecvBlockInfo{
char* buffer; char* buffer;
int nbLeaves; int nbLeaves;
}; };
RecvBlockInfo* recvBlockInfo = new RecvBlockInfo[nbProcs]; RecvBlockInfo* recvBlockInfo = new RecvBlockInfo[nbProcs];
int nbBlocksToRecv = 0; int nbBlocksToRecv = 0;
if(correctLeftLeavesNumber < currentLeafsOnMyLeft || currentRightLeafIdx < correctRightLeavesIndex){ if(correctLeftLeavesNumber < currentLeafsOnMyLeft || currentRightLeafIdx < correctRightLeavesIndex){
FSize iterCorrectLeafIdx = correctLeftLeavesNumber; FSize iterCorrectLeafIdx = correctLeftLeavesNumber;
int idxProcToProceed = 0; int idxProcToProceed = 0;
while(iterCorrectLeafIdx < correctRightLeavesIndex){ while(iterCorrectLeafIdx < correctRightLeavesIndex){
if(currentLeafsOnMyLeft <= iterCorrectLeafIdx && iterCorrectLeafIdx < currentRightLeafIdx){ if(currentLeafsOnMyLeft <= iterCorrectLeafIdx && iterCorrectLeafIdx < currentRightLeafIdx){
//// TODO REMOVE WHEN DEBUG printf("%d] currentLeafsOnMyLeft %llu iterCorrectLeafIdx %llu iterCorrectLeafIdx %llu currentRightLeafIdx %llu\n", //// TODO REMOVE WHEN DEBUG printf("%d] currentLeafsOnMyLeft %llu iterCorrectLeafIdx %llu iterCorrectLeafIdx %llu currentRightLeafIdx %llu\n",
//// TODO REMOVE WHEN DEBUG myRank, currentLeafsOnMyLeft, iterCorrectLeafIdx, iterCorrectLeafIdx, currentRightLeafIdx); //// TODO REMOVE WHEN DEBUG myRank, currentLeafsOnMyLeft, iterCorrectLeafIdx, iterCorrectLeafIdx, currentRightLeafIdx);
iterCorrectLeafIdx = currentRightLeafIdx; iterCorrectLeafIdx = currentRightLeafIdx;
idxProcToProceed = myRank + 1; idxProcToProceed = myRank + 1;
} }
else{ else{
//// TODO REMOVE WHEN DEBUG printf("%d] currentLeafsOnMyLeft %llu iterCorrectLeafIdx %llu iterCorrectLeafIdx %llu currentRightLeafIdx %llu correctRightLeavesIndex %llu\n", //// TODO REMOVE WHEN DEBUG printf("%d] currentLeafsOnMyLeft %llu iterCorrectLeafIdx %llu iterCorrectLeafIdx %llu currentRightLeafIdx %llu correctRightLeavesIndex %llu\n",
//// TODO REMOVE WHEN DEBUG myRank, currentLeafsOnMyLeft, iterCorrectLeafIdx, iterCorrectLeafIdx, currentRightLeafIdx, correctRightLeavesIndex); //// TODO REMOVE WHEN DEBUG myRank, currentLeafsOnMyLeft, iterCorrectLeafIdx, iterCorrectLeafIdx, currentRightLeafIdx, correctRightLeavesIndex);
while(leavesOffsetPerProc[idxProcToProceed+1] <= iterCorrectLeafIdx){ while(leavesOffsetPerProc[idxProcToProceed+1] <= iterCorrectLeafIdx){
//// TODO REMOVE WHEN DEBUG printf("%d] leavesOffsetPerProc[%d+1] %llu iterCorrectLeafIdx %lld\n", //// TODO REMOVE WHEN DEBUG printf("%d] leavesOffsetPerProc[%d+1] %llu iterCorrectLeafIdx %lld\n",
//// TODO REMOVE WHEN DEBUG myRank, idxProcToProceed, leavesOffsetPerProc[idxProcToProceed+1], iterCorrectLeafIdx); //// TODO REMOVE WHEN DEBUG myRank, idxProcToProceed, leavesOffsetPerProc[idxProcToProceed+1], iterCorrectLeafIdx);
idxProcToProceed += 1; idxProcToProceed += 1;
} }
const int nbLeafToReceive = FMath::Min(leavesOffsetPerProc[idxProcToProceed+1], int(correctRightLeavesIndex)) - int(iterCorrectLeafIdx); const int nbLeafToReceive = FMath::Min(leavesOffsetPerProc[idxProcToProceed+1], int(correctRightLeavesIndex)) - int(iterCorrectLeafIdx);
FSize nbParticlesToReceive = 0; FSize nbParticlesToReceive = 0;
for(int idxLeaf = 0 ; idxLeaf < nbLeafToReceive ; ++idxLeaf){ for(int idxLeaf = 0 ; idxLeaf < nbLeafToReceive ; ++idxLeaf){
nbParticlesToReceive += numberOfParticlesPerLeaf[idxLeaf + iterCorrectLeafIdx]; nbParticlesToReceive += numberOfParticlesPerLeaf[idxLeaf + iterCorrectLeafIdx];
} }
FSize bytesToRecv = (sizeof(ParticleClass)*nbParticlesToReceive) + sizeof(int)*nbLeafToReceive; FSize bytesToRecv = (sizeof(ParticleClass)*nbParticlesToReceive) + sizeof(int)*nbLeafToReceive;
char* bufferToReceive = new char[bytesToRecv]; char* bufferToReceive = new char[bytesToRecv];
//// TODO REMOVE WHEN DEBUG printf("Proc :: %d recv %d bytes to %d (from leaf %d to %d)\n", //// TODO REMOVE WHEN DEBUG printf("Proc :: %d recv %d bytes to %d (from leaf %d to %d)\n",
//// TODO REMOVE WHEN DEBUG myRank, bytesToRecv, idxProcToProceed, iterCorrectLeafIdx, iterCorrectLeafIdx + nbLeafToReceive); //// TODO REMOVE WHEN DEBUG myRank, bytesToRecv, idxProcToProceed, iterCorrectLeafIdx, iterCorrectLeafIdx + nbLeafToReceive);
MPI_Irecv(bufferToReceive, int(bytesToRecv), MPI_BYTE, idxProcToProceed, int(iterCorrectLeafIdx), MPI_Irecv(bufferToReceive, int(bytesToRecv), MPI_BYTE, idxProcToProceed, int(iterCorrectLeafIdx),
communicator.getComm(), &requests[counterRequest++]); communicator.getComm(), &requests[counterRequest++]);
recvBlockInfo[nbBlocksToRecv].buffer = bufferToReceive; recvBlockInfo[nbBlocksToRecv].buffer = bufferToReceive;
recvBlockInfo[nbBlocksToRecv].nbLeaves = nbLeafToReceive; recvBlockInfo[nbBlocksToRecv].nbLeaves = nbLeafToReceive;
nbBlocksToRecv += 1; nbBlocksToRecv += 1;
iterCorrectLeafIdx += nbLeafToReceive; iterCorrectLeafIdx += nbLeafToReceive;
} }
} }
} }
//// TODO REMOVE WHEN DEBUG printf("%d Wait!\n", myRank); //// TODO REMOVE WHEN DEBUG printf("%d Wait!\n", myRank);
MPI_Waitall(counterRequest, requests, MPI_STATUSES_IGNORE); MPI_Waitall(counterRequest, requests, MPI_STATUSES_IGNORE);
//// TODO REMOVE WHEN DEBUG printf("%d Done!\n", myRank); //// TODO REMOVE WHEN DEBUG printf("%d Done!\n", myRank);
int idxBlockRecvInLeft = 0; int idxBlockRecvInLeft = 0;
if(correctLeftLeavesNumber < currentLeafsOnMyLeft){ if(correctLeftLeavesNumber < currentLeafsOnMyLeft){
const int nbLeavesRecv = int(FMath::Min(currentLeafsOnMyLeft, correctRightLeavesIndex) - correctLeftLeavesNumber); const int nbLeavesRecv = int(FMath::Min(currentLeafsOnMyLeft, correctRightLeavesIndex) - correctLeftLeavesNumber);
//// TODO REMOVE WHEN DEBUG printf("%d] has receive %d from left\n", myRank, nbLeavesRecv); //// TODO REMOVE WHEN DEBUG printf("%d] has receive %d from left\n", myRank, nbLeavesRecv);
int idxLeaf = 0; int idxLeaf = 0;
while(idxLeaf < nbLeavesRecv){ while(idxLeaf < nbLeavesRecv){
//// TODO REMOVE WHEN DEBUG printf("%d] block %d has %d leaves\n", myRank, idxBlockRecvInLeft, recvBlockInfo[idxBlockRecvInLeft].nbLeaves); //// TODO REMOVE WHEN DEBUG printf("%d] block %d has %d leaves\n", myRank, idxBlockRecvInLeft, recvBlockInfo[idxBlockRecvInLeft].nbLeaves);
size_t offsetBuffer = 0; size_t offsetBuffer = 0;
for(int idxLeafInBlock = 0 ; idxLeafInBlock < recvBlockInfo[idxBlockRecvInLeft].nbLeaves ; ++idxLeafInBlock){ for(int idxLeafInBlock = 0 ; idxLeafInBlock < recvBlockInfo[idxBlockRecvInLeft].nbLeaves ; ++idxLeafInBlock){
const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&recvBlockInfo[idxBlockRecvInLeft].buffer[offsetBuffer])); const int numberOfParticlesInThisLeaf = (*reinterpret_cast<int*>(&recvBlockInfo[idxBlockRecvInLeft].buffer[offsetBuffer]));
const ParticleClass*const particles = reinterpret_cast<ParticleClass*>(&recvBlockInfo[idxBlockRecvInLeft].buffer[offsetBuffer] + sizeof(int)); const ParticleClass*const particles = reinterpret_cast<ParticleClass*>(&recvBlockInfo[idxBlockRecvInLeft].buffer[offsetBuffer] + sizeof(int));
//// TODO REMOVE WHEN DEBUG printf("%d] block %d leaf %d has %d part\n", myRank, idxBlockRecvInLeft, idxLeafInBlock, numberOfParticlesInThisLeaf); //// TODO REMOVE WHEN DEBUG printf("%d] block %d leaf %d has %d part\n", myRank, idxBlockRecvInLeft, idxLeafInBlock, numberOfParticlesInThisLeaf);
for(int idxParticle = 0 ; idxParticle < numberOfParticlesInThisLeaf ; ++idxParticle){ for(int idxParticle = 0 ; idxParticle < numberOfParticlesInThisLeaf ; ++idxParticle){
particlesSaver->push(particles[idxParticle]); particlesSaver->push(particles[idxParticle]);
} }
offsetBuffer += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int)); offsetBuffer += (sizeof(ParticleClass)*numberOfParticlesInThisLeaf+sizeof(int));
} }
idxLeaf += recvBlockInfo[idxBlockRecvInLeft].nbLeaves; idxLeaf += recvBlockInfo[idxBlockRecvInLeft].nbLeaves;
delete[] recvBlockInfo[idxBlockRecvInLeft].buffer; delete[] recvBlockInfo[idxBlockRecvInLeft].buffer;
idxBlockRecvInLeft += 1; idxBlockRecvInLeft += 1;
} }
} }