Commit 86db170e authored by PIACIBELLO Cyrille's avatar PIACIBELLO Cyrille

New balance algorithm for building the parallel tree. A Balance repertory has...

New balance algorithm for building the parallel tree. A Balance repertory has been created, with an FAbastractBalanceAlgorithm providing what it's needed to be a BalanceAlgorithm
parent 63680e00
// ===================================================================================
// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Bérenger Bramas, Matthias Messner
// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
#ifndef FABSTRACTBALANCEALGORITHM_H
#define FABSTRACTBALANCEALGORITHM_H
/**
* @author Cyrille Piacibello
* @class FAbstractBalanceAlgorithm
*
* @brief This class provide the methods that are used to balance a
* tree FMpiTreeBuilder::EqualizeAndFillTree
*/
class FAbstractBalanceAlgorithm{
public:
virtual ~FAbstractBalanceAlgorithm(){
}
/**
* @brief Give the right leaves (ie the min) of the interval that
* will be handle by idxOfProc
* @param numberOfLeaves Total number of leaves that exist.
* @param numberOfPartPerLeaf Array of lenght numberOfLeaves containing the number of particles in each leaf
* @param numberOfPart Number of particles in the whole field
* @param idxOfLeaves Array of lenght numberOfLeaves containing the Morton Index of each Leaf
* @param numberOfProc Number of MPI processus that will handle the Octree
* @param idxOfProc Idx of the proc calling.
*/
virtual FSize getRight(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc) = 0;
/**
* @brief Give the Leaft leaves (ie the max) of the interval that
* will be handle by idxOfProc
* @param numberOfLeaves Total number of leaves that exist.
* @param numberOfPartPerLeaf Array of lenght numberOfLeaves containing the number of particles in each leaf
* @param numberOfPart Number of particles in the whole field
* @param idxOfLeaves Array of lenght numberOfLeaves containing the Morton Index of each Leaf
* @param numberOfProc Number of MPI processus that will handle the Octree
* @param idxOfProc Idx of the proc calling.
*/
virtual FSize getLeft(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc) = 0;
};
#endif //FABSTRACTBALANCEALGORITHM_H
// ===================================================================================
// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Bérenger Bramas, Matthias Messner
// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
#ifndef FLEAFBALANCE_H
#define FLEAFBALANCE_H
#include "./FAbstractBalanceAlgorithm.hpp"
#include "../Utils/FMath.hpp"
/**
* @author Cyrille Piacibello
* @class FLeafBalance
*
* @brief This class inherits from FAbstractBalanceAlgorithm. It
* provides balancing methods based on leaf numbers only.
*/
class FLeafBalance : public FAbstractBalanceAlgorithm{
public:
/**
* Does not need the number of particles. Just divide the leaves
* over processus
*/
FSize getRight(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc){
const double step = (double(numberOfLeaves) / double(numberOfProc));
const FSize res = FSize(FMath::Ceil(step * double(idxOfProc+1)));
if(res > numberOfLeaves) return numberOfLeaves;
else return res;
}
/**
* Does not need the number of particles. Just divide the leaves
* over processus
*/
FSize getLeft(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc){
const double step = (double(numberOfLeaves) / double(numberOfProc));
return FSize(FMath::Ceil(step * double(idxOfProc)));
}
};
#endif // FLEAFBALANCE_H
// ===================================================================================
// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Bérenger Bramas, Matthias Messner
// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
#ifndef FPARTICLESBALANCE_H
#define FPARTICLESBALANCE_H
#include "./FAbstractBalanceAlgorithm.hpp"
#include "../Utils/FMath.hpp"
/**
* @author Cyrille Piacibello
* @class FLeafBalance
*
* @brief This class inherits from FAbstractBalanceAlgorithm. It
* provides balancing methods based on particles distribution.
*/
class FParticlesBalance : public FAbstractBalanceAlgorithm{
public:
/**
* getRight interval based on particles distribution
*/
FSize getRight(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc){
int acc = 0;
FSize i = 0;
const double step = (double(numberOfPart) / double(numberOfProc));
FSize aimRight = FSize(FMath::Ceil(step * double(idxOfProc+1)));
if(aimRight > numberOfPart) aimRight = numberOfPart;
while(acc < aimRight){
acc+=numberOfPartPerLeaf[i];
++i;
}
if(FMath::Abs(aimRight-acc) < FMath::Abs(aimRight-(acc-numberOfPartPerLeaf[i]))) return i;
else
return i-1;
}
/**
* get left interval based on particles distribution
*/
FSize getLeft(const FSize numberOfLeaves, const int*numberOfPartPerLeaf, const FSize numberOfPart, const MortonIndex* idxOfLeaves,
const int numberOfProc, const int idxOfProc){
int acc = 0;
FSize i = 0;
const double step = (double(numberOfPart) / double(numberOfProc));
const FSize aimLeft = FSize(FMath::Ceil(step * double(idxOfProc)));
while (acc < aimLeft){
acc+=numberOfPartPerLeaf[i];
++i;
}
if(FMath::Abs(aimLeft-acc) < FMath::Abs(aimLeft-(acc-numberOfPartPerLeaf[i]))) return i;
else
return i-1;
}
};
#endif // FPARTICLESBALANCE_H
......@@ -23,6 +23,8 @@
#include "../Utils/FMemUtils.hpp"
#include "../Utils/FTrace.hpp"
#include "../BalanceTree/FLeafBalance.hpp"
/** This class manage the loading of particles for the mpi version.
* It use a BinLoader and then sort the data with a parallel quick sort
* or bitonic sort.
......@@ -265,6 +267,7 @@ private:
++(*writeCounter);
}
}
delete [] workingArray;
workingArray = 0;
......@@ -279,7 +282,9 @@ private:
/** Put the interval into a tree */
template <class ContainerClass>
static void EqualizeAndFillTree(const FMpi::FComm& communicator, ContainerClass* particlesSaver,
char*& leavesArray, FSize& nbLeavesInIntervals ){
char*& leavesArray, FSize& nbLeavesInIntervals, FAbstractBalanceAlgorithm * balancer){
FTRACE( FTrace::FFunction functionTrace(__FUNCTION__, "Loader to Tree" , __FILE__ , __LINE__) );
const int rank = communicator.processId();
const int nbProcs = communicator.processCount();
......@@ -289,6 +294,45 @@ private:
FSize leavesPerProcs[nbProcs];
memset(leavesPerProcs, 0, sizeof(int) * nbProcs);
MPI_Allgather(&nbLeavesInIntervals, 1, MPI_LONG_LONG, leavesPerProcs, 1, MPI_LONG_LONG, communicator.getComm());
printf("Proc : %d : Currently have %lld leaves \n",rank,currentNbLeafs);
//Start working THERE
//We need the max number of leafs over th procs
FSize maxNbOfLeaf=0;
for(int i = 0 ; i< nbProcs ; ++i){
if(maxNbOfLeaf < leavesPerProcs[i]) maxNbOfLeaf=leavesPerProcs[i];
}
//Creation of an array to store how many parts are in each leaf
int * arrayToLeafs = new int[nbProcs*maxNbOfLeaf];
memset(arrayToLeafs,0,sizeof(int)*maxNbOfLeaf*nbProcs);
//Just indirection for simplifying
int * myParts = &arrayToLeafs[rank*maxNbOfLeaf];
//Loop over leafArray to fill myParts
FSize remainingLeafsToVisit = nbLeavesInIntervals;
long unsigned int firstIdx = 0;
while(remainingLeafsToVisit > 0){
int* intTab = reinterpret_cast<int*>(&leavesArray[firstIdx]);
//ParticleClass* tab = reinterpret_cast<ParticleClass*>(&leavesArray[firstIdx+sizeof(int)]);
//printf("Leaf %lld contains %d :: \n",nbLeavesInIntervals-remainingLeafsToVisit,intTab[0]);
for(int k =0; k<intTab[0] ; ++k){
myParts[nbLeavesInIntervals-remainingLeafsToVisit]++;
}
firstIdx += sizeof(ParticleClass)*intTab[0]+sizeof(int);
remainingLeafsToVisit--;
}
// {//TODO delete, because of uselessness
// if(rank==0){
// for(int k=0 ; k<nbLeavesInIntervals ; ++k){
// printf("%d \t",myParts[k]);
// }
// printf("\n");
// }
// }
MPI_Allgather(myParts,(int)maxNbOfLeaf,MPI_INT,arrayToLeafs,(int)maxNbOfLeaf,MPI_INT,communicator.getComm());
// Count the number of leaves on each side
FSize currentLeafsOnMyLeft = 0;
......@@ -301,8 +345,24 @@ private:
// So we can know the number of total leafs and
// the number of leaves each procs must have at the end
const FSize totalNbLeaves = (currentLeafsOnMyLeft + currentNbLeafs + currentLeafsOnMyRight);
const FSize correctLeftLeavesNumber = communicator.getLeft(totalNbLeaves);
const FSize correctRightLeavesIndex = communicator.getRight(totalNbLeaves);
int * leavesContents = new int[totalNbLeaves];
int idx = 0;
FSize totParts = 0;
for(int k=0 ; k<maxNbOfLeaf*nbProcs ; ++k){
if(arrayToLeafs[k]){
leavesContents[idx] = arrayToLeafs[k];
totParts += FSize(arrayToLeafs[k]);
idx++;
}
}
delete[] arrayToLeafs;
MortonIndex * notUsed = 0;
const FSize correctLeftLeavesNumber = balancer->getLeft( totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,rank);
const FSize correctRightLeavesIndex = balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,rank);
printf("Proc [%d] :: will work from leaf %lld \t to leaf %lld \n",rank,correctLeftLeavesNumber,correctRightLeavesIndex);
// This will be used for the all to all
int leavesToSend[nbProcs];
......@@ -325,7 +385,7 @@ private:
// Find the first proc that need my data
int idxProc = rank - 1;
while( idxProc > 0 ){
const FSize thisProcRight = communicator.getOtherRight(totalNbLeaves, idxProc - 1);
const FSize thisProcRight = balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc - 1);
// Go to left until proc-1 has a right index lower than my current left
if( thisProcRight < currentLeafsOnMyLeft){
break;
......@@ -335,8 +395,7 @@ private:
// Count data for this proc
int ICanGive = int(currentNbLeafs);
leavesToSend[idxProc] = int(FMath::Min(communicator.getOtherRight(totalNbLeaves, idxProc), totalNbLeaves - currentLeafsOnMyRight)
- FMath::Max( currentLeafsOnMyLeft , communicator.getOtherLeft(totalNbLeaves, idxProc)));
leavesToSend[idxProc] = int(FMath::Min(balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc),totalNbLeaves - currentLeafsOnMyRight) - FMath::Max( currentLeafsOnMyLeft , balancer->getLeft(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc)));
{
bytesOffset[idxProc] = 0;
for(FSize idxLeaf = 0 ; idxLeaf < leavesToSend[idxProc] ; ++idxLeaf){
......@@ -349,7 +408,7 @@ private:
// count data to other proc
while(idxProc < rank && ICanGive){
leavesToSend[idxProc] = int(FMath::Min( communicator.getOtherRight(totalNbLeaves, idxProc) - communicator.getOtherLeft(totalNbLeaves, idxProc), FSize(ICanGive)));
leavesToSend[idxProc] = int(FMath::Min( balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc) - balancer->getLeft(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc), FSize(ICanGive)));
bytesOffset[idxProc] = int(currentIntervalPosition);
for(FSize idxLeaf = 0 ; idxLeaf < leavesToSend[idxProc] ; ++idxLeaf){
......@@ -386,8 +445,8 @@ private:
// Find the last proc on the right that need my data
int idxProc = rank + 1;
while( idxProc < nbProcs ){
const FSize thisProcLeft = communicator.getOtherLeft(totalNbLeaves, idxProc);
const FSize thisProcRight = communicator.getOtherRight(totalNbLeaves, idxProc);
const FSize thisProcLeft = balancer->getLeft(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc);
const FSize thisProcRight = balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc);
// Progress until the proc+1 has its left index upper to my current right
if( thisProcLeft < currentLeafsOnMyLeft || (totalNbLeaves - currentLeafsOnMyRight) < thisProcRight){
break;
......@@ -397,8 +456,7 @@ private:
// Count the data
int ICanGive = int(currentLeafsOnMyLeft + currentNbLeafs - correctRightLeavesIndex);
leavesToSend[idxProc] = int(FMath::Min(communicator.getOtherRight(totalNbLeaves, idxProc) , (totalNbLeaves - currentLeafsOnMyRight))
- FMath::Max(communicator.getOtherLeft(totalNbLeaves, idxProc), currentLeafsOnMyLeft) );
leavesToSend[idxProc] = int(FMath::Min(balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc) , (totalNbLeaves - currentLeafsOnMyRight)) - FMath::Max(balancer->getLeft(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc), currentLeafsOnMyLeft) );
{
bytesOffset[idxProc] = int(currentIntervalPosition);
......@@ -412,7 +470,7 @@ private:
// Now Count the data to other
while(idxProc < nbProcs && ICanGive){
leavesToSend[idxProc] = int(FMath::Min( communicator.getOtherRight(totalNbLeaves, idxProc) - communicator.getOtherLeft(totalNbLeaves, idxProc), FSize(ICanGive)));
leavesToSend[idxProc] = int(FMath::Min( balancer->getRight(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs, idxProc) - balancer->getLeft(totalNbLeaves,leavesContents,totParts,notUsed,nbProcs,idxProc), FSize(ICanGive)));
bytesOffset[idxProc] = int(currentIntervalPosition);
for(FSize idxLeaf = 0 ; idxLeaf < leavesToSend[idxProc] ; ++idxLeaf){
......@@ -504,7 +562,7 @@ public:
template <class ContainerClass>
static void ArrayToTree(const FMpi::FComm& communicator, const ParticleClass array[], const FSize size,
const FPoint& boxCenter, const FReal boxWidth, const int treeHeight,
ContainerClass* particleSaver, const SortingType type = QuickSort){
ContainerClass* particleSaver, FAbstractBalanceAlgorithm* balancer,const SortingType type = QuickSort){
IndexedParticle* particlesArray = 0;
FSize particlesSize = 0;
......@@ -515,7 +573,7 @@ public:
FSize leavesSize = 0;
MergeLeaves(communicator, particlesArray, particlesSize, &leavesArray, &leavesSize);
EqualizeAndFillTree(communicator, particleSaver, leavesArray, leavesSize);
EqualizeAndFillTree(communicator, particleSaver, leavesArray, leavesSize, balancer);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment