Commit cce925b8 authored by BRAMAS Berenger's avatar BRAMAS Berenger

add a test to guarantee that the particles are sorted and debug the balancing stage

parent 2afd4369
......@@ -355,12 +355,16 @@ public:
for(int idxProc = 0 ; idxProc < nbProcs ; ++idxProc){
allObjectives[idxProc].first = balancer->getLeft(totalNumberOfLeavesInSimulation,nbProcs,idxProc);
allObjectives[idxProc].second = balancer->getRight(totalNumberOfLeavesInSimulation,nbProcs,idxProc);
if(idxProc != 0) FAssertLF(allObjectives[idxProc].first == allObjectives[idxProc-1].second);
}
// Ask for the pack to send
std::pair<size_t, size_t> myCurrentInter = {diffNumberOfLeavesPerProc[myRank], diffNumberOfLeavesPerProc[myRank+1]};
const std::vector<FEqualize::Package> packsToSend = FEqualize::GetPackToSend(myCurrentInter, allObjectives);
FAssertLF((currentNbLeaves == 0 && packsToSend.size() == 0) ||
(packsToSend.size() && FSize(packsToSend[packsToSend.size()-1].elementTo) == currentNbLeaves));
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Get my interval (" << packsToSend.size() << ")\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send data\n"; FLog::Controller.flush(); );
......@@ -369,22 +373,34 @@ public:
requestsNbParts.reserve(packsToSend.size());
// Send every thing except for me or if size == 0
FSize totalSend = 0;
FSize sendToMe = 0;
for(unsigned int idxPack = 0; idxPack< packsToSend.size() ; ++idxPack){
const FEqualize::Package& pack = packsToSend[idxPack];
if(idxPack != 0) FAssertLF(packsToSend[idxPack].elementFrom == packsToSend[idxPack-1].elementTo);
const long long int nbPartsPerPackToSend = leavesOffsetInParticles[pack.elementTo]-leavesOffsetInParticles[pack.elementFrom];
totalSend += nbPartsPerPackToSend;
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
// If not to me and if there is something to send
const long long int nbPartsPerPackToSend = leavesOffsetInParticles[pack.elementTo]-leavesOffsetInParticles[pack.elementFrom];
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-send to " << pack.idProc << " nb " << nbPartsPerPackToSend << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-send to " << pack.idProc << " nb " << nbPartsPerPackToSend
<< " from " << pack.elementFrom << " to " << pack.elementTo << " \n"; FLog::Controller.flush(); );
// Send the size of the data
requestsNbParts.emplace_back();
FMpi::MpiAssert(MPI_Isend(&nbPartsPerPackToSend,1,MPI_LONG_LONG_INT,pack.idProc,
FMpi::TagExchangeIndexs, communicator.getComm(), &requestsNbParts.back()),__LINE__);
}
else {
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
sendToMe = nbPartsPerPackToSend;
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack
<< " from " << pack.elementFrom << " to " << pack.elementTo << " \n"; FLog::Controller.flush(); );
}
}
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send done \n"; FLog::Controller.flush(); );
// Ensure everything has been proceed
FAssertLF(totalSend == currentNbParts);
// Compute the current intervals
std::vector< std::pair<size_t,size_t> > allCurrentIntervals;
......@@ -403,6 +419,9 @@ public:
std::unique_ptr<FSize[]> nbPartsPerPackToRecv(new FSize[packsToRecv.size()]);
for(unsigned int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
if(idxPack != 0) FAssertLF(packsToRecv[idxPack].elementFrom == packsToRecv[idxPack-1].elementTo);
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-recv from " << pack.idProc << " \n"; FLog::Controller.flush(); );
// We need to know how much particles to receive
......@@ -412,11 +431,13 @@ public:
}
else{
if(pack.idProc == myRank){
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip recv " << idxPack << " \n"; FLog::Controller.flush(); );
// Take my own data
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
const FSize nbLeavesToCopy = pack.elementTo-pack.elementFrom;
nbPartsPerPackToRecv[idxPack] = leavesOffsetInParticles[sourcePosition+nbLeavesToCopy] - leavesOffsetInParticles[sourcePosition];
FAssertLF(nbPartsPerPackToRecv[idxPack] == sendToMe);
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip recv " <<
idxPack << " nb " << nbPartsPerPackToRecv[idxPack] << " \n"; FLog::Controller.flush(); );
}
else{
// Nothing to receive from this so avoid communication
......@@ -436,11 +457,13 @@ public:
for(unsigned int idxPack = 0; idxPack< packsToSend.size() ; ++idxPack){
const FEqualize::Package& pack = packsToSend[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
const long long int nbPartsPerPackToSend = leavesOffsetInParticles[pack.elementTo]-leavesOffsetInParticles[pack.elementFrom];
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] send to "
<< pack.idProc << " nb " << (pack.elementTo-pack.elementFrom) << " \n"; FLog::Controller.flush(); );
<< pack.idProc << " nb " << nbPartsPerPackToSend << " \n"; FLog::Controller.flush(); );
FMpi::ISendSplit(&particlesArrayInLeafOrder[pack.elementFrom],
(pack.elementTo - pack.elementFrom),
FMpi::ISendSplit(&particlesArrayInLeafOrder[leavesOffsetInParticles[pack.elementFrom]],
nbPartsPerPackToSend,
pack.idProc,
FMpi::TagExchangeIndexs + 1,
communicator,
......@@ -451,7 +474,9 @@ public:
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] barrier after all send \n"; FLog::Controller.flush(); );
////////////////////////////////////////////////////////////////
// Count the number of leaf to receive
////////////////////////////////////////////////////////////////
FSize totalPartsToReceive = 0;
for(unsigned int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
totalPartsToReceive += nbPartsPerPackToRecv[idxPack];
......@@ -459,28 +484,36 @@ public:
std::unique_ptr<ParticleClass[]> particlesRecvBuffer(new ParticleClass[totalPartsToReceive]);
////////////////////////////////////////////////////////////////
// Post all the receive and copy mine
// it is based on the nbPartsPerPackToRecv array
////////////////////////////////////////////////////////////////
if(totalPartsToReceive){
FSize offsetToRecv = 0;
for(unsigned int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
// If it is not from me
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv from "
<< pack.idProc << " nb " << nbPartsPerPackToRecv[idxPack] << "\n"; FLog::Controller.flush(); );
<< pack.idProc << " nb " << nbPartsPerPackToRecv[idxPack] << " from " << pack.elementFrom << "\n"; FLog::Controller.flush(); );
FAssertLF(pack.elementTo <= size_t(totalPartsToReceive));
FMpi::IRecvSplit(&particlesRecvBuffer[pack.elementFrom],
(pack.elementTo - pack.elementFrom),
// We store from offset, and use nbPartsPerPackToRecv has the number
FMpi::IRecvSplit(&particlesRecvBuffer[offsetToRecv],
nbPartsPerPackToRecv[idxPack],
pack.idProc,
FMpi::TagExchangeIndexs + 1,
communicator,
&requestsParts);
}
// it is from me, just copy
else if(pack.idProc == myRank){
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] copy "
<< idxPack << " nb " << nbPartsPerPackToRecv[idxPack] << " from " << pack.elementFrom << " \n"; FLog::Controller.flush(); );
// Copy my particles
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
// We store from offset, and use nbPartsPerPackToRecv has the number
// The reading position is the offset of the first leaf we own
memcpy(&particlesRecvBuffer[offsetToRecv], &particlesArrayInLeafOrder[leavesOffsetInParticles[sourcePosition]],
nbPartsPerPackToRecv[idxPack]*sizeof(ParticleClass));
}
......@@ -522,6 +555,10 @@ public:
FLOG( FLog::Controller << "[" << communicator.processId() << "] Particles Distribution: " << "\t GetSortedParticlesFromArray is over (" << timer.tacAndElapsed() << "s)\n"; FLog::Controller.flush(); );
FLOG( timer.tic() );
// for(int idx = 0 ; idx < nbParticlesInArray ; ++idx){
// particleSaver->push(sortedParticlesArray[idx].particle);
// }
ParticleClass* particlesArrayInLeafOrder = nullptr;
FSize * leavesOffsetInParticles = nullptr;
FSize nbLeaves = 0;
......@@ -529,6 +566,10 @@ public:
MergeSplitedLeaves(communicator, sortedParticlesArray, &nbParticlesInArray, &leavesOffsetInParticles, &particlesArrayInLeafOrder, &nbLeaves);
delete[] sortedParticlesArray;
// for(int idx = 0 ; idx < nbParticlesInArray ; ++idx){
// particleSaver->push(particlesArrayInLeafOrder[idx]);
// }
FLOG( FLog::Controller << "[" << communicator.processId() << "] Particles Distribution: " << "\t MergeSplitedLeaves is over (" << timer.tacAndElapsed() << "s)\n"; FLog::Controller.flush(); );
FLOG( timer.tic() );
......
// ===================================================================================
// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Berenger Bramas
// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
// ==== CMAKE =====
// @FUSE_MPI
// ================
#include <iostream>
#include <cstdio>
#include <cstdlib>
#include "../../Src/Kernels/Rotation/FRotationCell.hpp"
#include "../../Src/Kernels/Rotation/FRotationKernel.hpp"
#include "../../Src/Components/FSimpleLeaf.hpp"
#include "../../Src/Kernels/P2P/FP2PParticleContainerIndexed.hpp"
#include "../../Src/Utils/FParameters.hpp"
#include "../../Src/Utils/FMemUtils.hpp"
#include "../../Src/Containers/FOctree.hpp"
#include "../../Src/Containers/FVector.hpp"
#include "../../Src/Files/FRandomLoader.hpp"
#include "../../Src/Files/FMpiTreeBuilder.hpp"
#include "../../Src/Core/FFmmAlgorithm.hpp"
#include "../../Src/Core/FFmmAlgorithmThread.hpp"
#include "../../Src/Core/FFmmAlgorithmThreadProc.hpp"
#include "../../Src/BalanceTree/FLeafBalance.hpp"
#include "../../Src/Utils/FParameterNames.hpp"
/**
* This program runs the FMM Algorithm Distributed with the Rotation kernel
*/
// Simply create particles and try the kernels
int main(int argc, char* argv[])
{
FHelpDescribeAndExit(argc, argv,
"Test with MPI the chebyshev FMM and compare it to the direct computation for debugging purpose.",
FParameterDefinitions::NbParticles, FParameterDefinitions::OctreeHeight,
FParameterDefinitions::OctreeSubHeight, FParameterDefinitions::NbThreads);
typedef double FReal;
FMpi app(argc,argv);
const FSize nbParticles = FParameters::getValue(argc,argv, FParameterDefinitions::NbParticles.options, 10000000ULL);
const unsigned int TreeHeight = FParameters::getValue(argc, argv, FParameterDefinitions::OctreeHeight.options, 5);
FTic time;
std::cout << ">> This executable has to be used to test Proc Rotation Algorithm. \n";
// init particles position and physical value
struct TestParticle{
FSize idxPart;
FPoint<FReal> position;
FReal physicalValue;
const FPoint<FReal>& getPosition(){
return position;
}
};
// open particle file
std::cout << "Creating : " << nbParticles << "\n" << std::endl;
time.tic();
const FSize totalNbParticles = nbParticles*app.global().processCount();
TestParticle* particles = new TestParticle[totalNbParticles];
memset(particles,0,(unsigned int) (sizeof(TestParticle)*totalNbParticles));
for(int idxProc = 0 ; idxProc < app.global().processCount() ; ++idxProc){
FRandomLoader<FReal> loader(nbParticles, 1.0, FPoint<FReal>(0,0,0), idxProc);
for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
loader.fillParticle(&particles[idxPart + idxProc*nbParticles].position);
particles[idxPart + idxProc*nbParticles].physicalValue = 1.0;
particles[idxPart + idxProc*nbParticles].idxPart = idxPart + idxProc*nbParticles;
}
}
FVector<TestParticle> finalParticles;
FLeafBalance balancer;
FMpiTreeBuilder< FReal,TestParticle >::DistributeArrayToContainer(app.global(),&particles[app.global().processId()*nbParticles],
nbParticles,
FPoint<FReal>(0,0,0),
1.0,TreeHeight,
&finalParticles, &balancer);
app.global().barrier();
std::cout << "Testing : " << finalParticles.getSize() << "\n" << std::endl;
for(FSize idxRes = 0 ; idxRes < finalParticles.getSize() ; ++idxRes){
FAssertLF(0 <= finalParticles[idxRes].idxPart, "idxRes ", idxRes, " finalParticles[idxRes].idxPart ", finalParticles[idxRes].idxPart);
FAssertLF(finalParticles[idxRes].idxPart < totalNbParticles, "idxRes ", idxRes, " finalParticles[idxRes].idxPart ", finalParticles[idxRes].idxPart);
const TestParticle correctPart = particles[finalParticles[idxRes].idxPart];
const TestParticle testPart = finalParticles[idxRes];
FAssertLF(testPart.idxPart == correctPart.idxPart);
FAssertLF(testPart.position.getX() == correctPart.position.getX());
FAssertLF(testPart.position.getY() == correctPart.position.getY());
FAssertLF(testPart.position.getZ() == correctPart.position.getZ());
FAssertLF(testPart.physicalValue == correctPart.physicalValue);
}
std::cout << "Done\n" << std::endl;
app.global().barrier();
std::unique_ptr<int[]> particlesExist(new int[totalNbParticles]);
memset(particlesExist.get(), 0, sizeof(int)*totalNbParticles);
for(FSize idxRes = 0 ; idxRes < finalParticles.getSize() ; ++idxRes){
FAssertLF(particlesExist[finalParticles[idxRes].idxPart] == 0);
particlesExist[finalParticles[idxRes].idxPart] = 1;
}
std::unique_ptr<int[]> particlesReduced(new int[totalNbParticles]);
memset(particlesReduced.get(), 0, sizeof(int)*totalNbParticles);
FAssert(totalNbParticles <= std::numeric_limits<int>::max());
FMpi::Assert(MPI_Allreduce(particlesExist.get(), particlesReduced.get(), int(totalNbParticles),
MPI_INT, MPI_SUM,
app.global().getComm()), __LINE__);
for(FSize idxPart = 0 ; idxPart < totalNbParticles ; ++idxPart){
FAssertLF(particlesReduced[idxPart] == 1, idxPart, " " , particlesReduced[idxPart]);
}
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment