Commit cc50a574 authored by PIACIBELLO Cyrille's avatar PIACIBELLO Cyrille
Browse files

Mpi tree builder is supposed to work fine

parent bdccb780
......@@ -36,7 +36,7 @@ public:
/**
* To know what to send to who.
* @param myCurrentInterval current process interval
* @param allObjectives the intevals that each process should have (in ascendant order, exclusive)
* @param allObjectives the intervals that each process should have (in ascendant order, exclusive)
* @return the package that the current worker should sent to others
*/
static std::vector<Package> GetPackToSend(const std::pair<size_t, size_t> myCurrentInterval,
......
......@@ -160,7 +160,7 @@ public:
//////////////////////////////////////////////////////////////////////////
static void MergeSplitedLeaves(const FMpi::FComm& communicator, IndexedParticle* workingArray, FSize* workingSize,
FSize ** leavesOffsetInParticles, ParticleClass** particlesArrayInLeafOrder, FSize* const leavesSize){
FSize ** leavesOffsetInParticles, ParticleClass** particlesArrayInLeafOrder, FSize* const leavesSize){
FTRACE( FTrace::FFunction functionTrace(__FUNCTION__, "Loader to Tree" , __FILE__ , __LINE__) );
const int myRank = communicator.processId();
const int nbProcs = communicator.processCount();
......@@ -331,33 +331,45 @@ public:
else{
// We need to know the number of leaves per procs
std::unique_ptr<FSize[]> numberOfLeavesPerProc(new FSize[nbProcs]);
MPI_Allgather((FSize*)&currentNbLeaves, 1, MPI_LONG_LONG_INT, numberOfLeavesPerProc.get(), 1, MPI_LONG_LONG_INT, communicator.getComm());
MPI_Allgather((FSize*)&currentNbLeaves, 1, MPI_LONG_LONG_INT, numberOfLeavesPerProc.get(), 1, MPI_LONG_LONG_INT, communicator.getComm());
//prefix sum
std::unique_ptr<FSize[]> diffNumberOfLeavesPerProc(new FSize[nbProcs+1]);
diffNumberOfLeavesPerProc[0] = 0;
for(int idxProc = 0 ; idxProc < nbProcs ; ++idxProc ){
diffNumberOfLeavesPerProc[idxProc+1] = diffNumberOfLeavesPerProc[idxProc] + numberOfLeavesPerProc[idxProc];
}
const FSize totalNumberOfLeavesInSimulation = numberOfLeavesPerProc[nbProcs];
// Compute the objective interval
}
const FSize totalNumberOfLeavesInSimulation = diffNumberOfLeavesPerProc[nbProcs];
// Compute the objective interval
std::vector< std::pair<size_t,size_t> > allObjectives;
allObjectives.resize(nbProcs);
for(int idxProc = 0 ; idxProc < nbProcs ; ++idxProc){
allObjectives[idxProc].first = balancer->getLeft(totalNumberOfLeavesInSimulation,NULL,0,0,nbProcs,idxProc);
allObjectives[idxProc].second = balancer->getRight(totalNumberOfLeavesInSimulation,NULL,0,0,nbProcs,idxProc);
}
// Ask for the pack to send
std::pair<size_t, size_t> myCurrentInter = {diffNumberOfLeavesPerProc[myRank], diffNumberOfLeavesPerProc[myRank+1]};
const std::vector<FEqualize::Package> packsToSend = FEqualize::GetPackToSend(myCurrentInter, allObjectives);
std::unique_ptr<FSize[]> nbPartsPerPackToSend(new FSize[packsToSend.size()]);
// Store the requests
std::vector<MPI_Request> requests;
std::vector<MPI_Request> requestsParts;
std::vector<MPI_Request> requestsNbParts;
// Send every thing except for me or if size == 0
for(const FEqualize::Package& pack : packsToSend){
for(int idxPack = 0; idxPack< packsToSend.size() ; ++idxPack){
const FEqualize::Package& pack = packsToSend[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
requests.emplace_back();
MPI_Isend((ParticleClass*)&particlesArrayInLeafOrder[leavesOffsetInParticles[pack.elementFrom]], pack.elementTo-pack.elementFrom,
MPI_BYTE, pack.idProc, FMpi::TagExchangeIndexs, communicator.getComm(), &requests.back());
//First, we need to send the size of the leaves we will send
nbPartsPerPackToSend[idxPack] = leavesOffsetInParticles[pack.elementTo]-leavesOffsetInParticles[pack.elementFrom];
requestsNbParts.emplace_back();
MPI_Isend(&nbPartsPerPackToSend[idxPack],1,MPI_LONG_LONG_INT,pack.idProc,
FMpi::TagExchangeIndexs+1, communicator.getComm(), &requestsNbParts.back());
requestsParts.emplace_back();
MPI_Isend((ParticleClass*)&particlesArrayInLeafOrder[leavesOffsetInParticles[pack.elementFrom]],
sizeof(ParticleClass)*nbPartsPerPackToSend[idxPack],
MPI_BYTE, pack.idProc, FMpi::TagExchangeIndexs, communicator.getComm(), &requestsParts.back());
}
}
// Compute the current intervals
......@@ -370,33 +382,57 @@ public:
// Ask the packs to receive to fill my objective
std::pair<size_t, size_t> myObjective = allObjectives[myRank];
const std::vector<FEqualize::Package> packsToRecv = FEqualize::GetPackToRecv(myObjective, allCurrentIntervals);
// Count the number of element to receive
FSize totalParticlesToReceive = 0;
for(const FEqualize::Package& pack : packsToRecv){
totalParticlesToReceive += (pack.elementTo-pack.elementFrom);
}
// Count the number of parts to receive
std::unique_ptr<FSize[]> nbPartsPerPackToRecv(new FSize[packsToRecv.size()]);
for(int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
requestsNbParts.emplace_back();
MPI_Irecv(&nbPartsPerPackToRecv[idxPack], 1, MPI_LONG_LONG_INT, pack.idProc,
FMpi::TagExchangeIndexs+1, communicator.getComm(), &requestsNbParts.back());
}
else{
if(pack.idProc == myRank){
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
const FSize nbLeavesToCopy = pack.elementTo-pack.elementFrom;
nbPartsPerPackToRecv[idxPack] = leavesOffsetInParticles[sourcePosition+nbLeavesToCopy] - leavesOffsetInParticles[sourcePosition];
}
}
}
MPI_Waitall(requestsNbParts.size(), requestsNbParts.data(), MPI_STATUSES_IGNORE);
// Count the number of leaf to receive
FSize totalPartsToReceive = 0;
for(int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
totalPartsToReceive += nbPartsPerPackToRecv[idxPack];
}
std::vector<ParticleClass> particlesRecvBuffer;
// Post all the receive and copy mine
if(totalParticlesToReceive){
particlesRecvBuffer.resize(totalParticlesToReceive);
for(const FEqualize::Package& pack : packsToRecv){
if(totalPartsToReceive){
particlesRecvBuffer.resize(totalPartsToReceive);
FSize offsetToRecv = 0;
for(int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
requests.emplace_back();
MPI_Irecv(&particlesRecvBuffer[pack.elementFrom], pack.elementTo-pack.elementFrom, MPI_BYTE, pack.idProc,
FMpi::TagExchangeIndexs, communicator.getComm(), &requests.back());
}
requestsParts.emplace_back();
MPI_Irecv(&particlesRecvBuffer[offsetToRecv], sizeof(ParticleClass)*nbPartsPerPackToRecv[idxPack], MPI_BYTE, pack.idProc,
FMpi::TagExchangeIndexs, communicator.getComm(), &requestsParts.back());
}
else if(pack.idProc == myRank){
// Copy my particles
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
memcpy(&particlesRecvBuffer[pack.elementFrom], &particlesArrayInLeafOrder[leavesOffsetInParticles[sourcePosition]],
(pack.elementTo-pack.elementFrom)*sizeof(ParticleClass));
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
memcpy(&particlesRecvBuffer[offsetToRecv], &particlesArrayInLeafOrder[leavesOffsetInParticles[sourcePosition]],
nbPartsPerPackToRecv[idxPack]*sizeof(ParticleClass));
}
offsetToRecv += nbPartsPerPackToRecv[idxPack];
}
}
// Finalize communication
MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
MPI_Waitall(requestsParts.size(), requestsParts.data(), MPI_STATUSES_IGNORE);
// Insert in the particle saver
for(FSize idPartsToStore = 0 ; idPartsToStore < particlesRecvBuffer.size() ; ++idPartsToStore){
......
......@@ -115,7 +115,7 @@ class TestMpiTreeBuilder : public FUTesterMpi< class TestMpiTreeBuilder> {
"UTest/DirectDouble.bfma");
//Let the choice there to test
//std::string filename(SCALFMMDataPath+parFile);
//std::string filename("../Data/unitCubeXYZQ100.bfma");
std::string filename("../Data/unitCubeXYZQ100.bfma");
int TreeHeight =4;
......@@ -231,7 +231,7 @@ class TestMpiTreeBuilder : public FUTesterMpi< class TestMpiTreeBuilder> {
FSize * leavesIndices = nullptr;
FSize leaveSize = 0;
FMpiTreeBuilder<TestParticle>::testMergeLeaves(app.global(),outputArray,&outputSize,&leavesIndices,&leavesArray,&leaveSize);
FMpiTreeBuilder<TestParticle>::MergeSplitedLeaves(app.global(),outputArray,&outputSize,&leavesIndices,&leavesArray,&leaveSize);
//Compare again the results with the output of std::qsort
......@@ -258,13 +258,12 @@ class TestMpiTreeBuilder : public FUTesterMpi< class TestMpiTreeBuilder> {
bool resultEqualize = true;
FMpiTreeBuilder<TestParticle>::testEqualizeAndFillTree(app.global(),&finalParticles,leavesIndices,leavesArray,leaveSize,outputSize,&balancer);
FMpiTreeBuilder<TestParticle>::EqualizeAndFillContainer(app.global(),&finalParticles,leavesIndices,leavesArray,leaveSize,outputSize,&balancer);
//Ok now count the Particles at the end of the Equalize
int finalNbPart = finalParticles.getSize();
int finalStart = 0;
MPI_Exscan(&finalNbPart,&finalStart,1,MPI_INT,MPI_SUM,app.global().getComm());
for (int k=0; k<finalNbPart ; k++){
if(finalParticles[k].indexInFile != arrayOfParticles[k+finalStart].indexInFile){
printf("Equalize :: Proc %d, k=[%d+%d] finalParticles : %lld,%lld, sortedArray %lld,%lld \n",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment