Commit 794015bd authored by Berenger Bramas's avatar Berenger Bramas

debug mpi from tinker branch feedback (it does not include the tinker part)

parent f2533aae
......@@ -164,6 +164,8 @@ int main(int argc, char* argv[])
tree.getBoxWidth(),
tree.getHeight(), &finalParticles,&balancer);
std::cout << "Local nb particles after sort "
<< finalParticles.getSize() << std::endl;
for(FSize idx = 0 ; idx < finalParticles.getSize(); ++idx){
tree.insert(finalParticles[idx].position,finalParticles[idx].indexInFile,finalParticles[idx].physicalValue);
}
......
......@@ -4,6 +4,9 @@ the more restrictive has to be used.
ScalFmm est régi par la licence CeCILL-C & LGPL, en cas de conflit
la plus restrictive prime.
Folders under Addons might have separate Licence, in such case
one can find a dedicated Licence file where appropriate.
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
......
......@@ -81,7 +81,10 @@ public:
pack.elementTo = Min(allObjectives[idxProc].second , myCurrentInterval.second) - myCurrentInterval.first;
// Next time give from the previous end
currentElement = pack.elementTo;
packToSend.push_back(pack);
if(pack.elementTo - pack.elementFrom != 0){
packToSend.push_back(pack);
}
// Progress
idxProc += 1;
}
......
This diff is collapsed.
......@@ -1685,6 +1685,10 @@ public:
* @param function
*/
void forEachLeaf(std::function<void(LeafClass*)> function){
if(isEmpty()){
return;
}
Iterator octreeIterator(this);
octreeIterator.gotoBottomLeft();
......@@ -1698,6 +1702,10 @@ public:
* @param function
*/
void forEachCell(std::function<void(CellClass*)> function){
if(isEmpty()){
return;
}
Iterator octreeIterator(this);
octreeIterator.gotoBottomLeft();
......@@ -1717,6 +1725,10 @@ public:
* @param function
*/
void forEachCellWithLevel(std::function<void(CellClass*,const int)> function){
if(isEmpty()){
return;
}
Iterator octreeIterator(this);
octreeIterator.gotoBottomLeft();
......@@ -1736,6 +1748,10 @@ public:
* @param function
*/
void forEachCellLeaf(std::function<void(CellClass*,LeafClass*)> function){
if(isEmpty()){
return;
}
Iterator octreeIterator(this);
octreeIterator.gotoBottomLeft();
......
This diff is collapsed.
This diff is collapsed.
......@@ -107,7 +107,7 @@ public:
* This class is used to gather the usual methods related to identifying an
* MPI communicator.
*/
class FComm : public FNoCopyable {
class FComm {
int rank; ///< rank related to the comm
int nbProc; ///< nb proc in this group
......@@ -130,6 +130,26 @@ public:
reset();
}
/// Constructor : duplicates the given communicator
FComm(const FComm& inCommunicator ) {
FMpi::Assert( MPI_Comm_dup(inCommunicator.communicator, &communicator), __LINE__ , "comm dup");
FMpi::Assert( MPI_Comm_group(communicator, &group), __LINE__ , "comm group");
reset();
}
FComm& operator=(const FComm& inCommunicator ) {
FMpi::Assert( MPI_Comm_free(&communicator), __LINE__ );
FMpi::Assert( MPI_Group_free(&group), __LINE__ );
FMpi::Assert( MPI_Comm_dup(inCommunicator.communicator, &communicator), __LINE__ , "comm dup");
FMpi::Assert( MPI_Comm_group(communicator, &group), __LINE__ , "comm group");
reset();
return *this;
}
/// Frees communicator and group
virtual ~FComm(){
FMpi::Assert( MPI_Comm_free(&communicator), __LINE__ );
......@@ -248,6 +268,35 @@ public:
delete[] procsIdArray ;
}
/** Change the group, create one groupd where processInGroup[i] != 0
* and another where processInGroup[i] == 0
*/
void groupReduce(const int processInGroup[]){
int * procsIdArray = new int [nbProc];
int counterNewGroup = 0;
for(int idxProc = 0 ;idxProc < nbProc ; ++idxProc){
if(processInGroup[rank] && processInGroup[idxProc]){
procsIdArray[counterNewGroup++] = idxProc;
}
else if(!processInGroup[rank] && !processInGroup[idxProc]){
procsIdArray[counterNewGroup++] = idxProc;
}
}
MPI_Group previousGroup = group;
FMpi::Assert( MPI_Group_incl(previousGroup, counterNewGroup , procsIdArray, &group), __LINE__ );
MPI_Comm previousComm = communicator;
FMpi::Assert( MPI_Comm_create(previousComm, group, &communicator), __LINE__ );
MPI_Comm_free(&previousComm);
MPI_Group_free(&previousGroup);
reset();
FAssertLF(nbProc == counterNewGroup);
delete[] procsIdArray ;
}
void barrier() const {
FMpi::Assert(MPI_Barrier(getComm()), __LINE__);
}
......
......@@ -93,14 +93,18 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
{
// Get the number of elements each proc should recv
IndexType totalRemainingElements = totalElements;
IndexType totalAvailableElements = totalElementToProceed;
for(int idxProc = firstProcToRecv; idxProc < lastProcToRecv ; ++idxProc){
const IndexType nbElementsAlreadyOwned = (inFromRightToLeft ? globalElementBalance[idxProc].lowerPart : globalElementBalance[idxProc].greaterPart);
const IndexType averageNbElementForRemainingProc = (totalRemainingElements)/(lastProcToRecv-idxProc);
totalRemainingElements -= nbElementsAlreadyOwned;
FAssertLF(totalRemainingElements >= 0);
if(nbElementsAlreadyOwned < averageNbElementForRemainingProc){
nbElementsToRecvPerProc[idxProc - firstProcToRecv] = (averageNbElementForRemainingProc - nbElementsAlreadyOwned);
if(nbElementsAlreadyOwned < averageNbElementForRemainingProc && totalAvailableElements){
nbElementsToRecvPerProc[idxProc - firstProcToRecv] = FMath::Min(totalAvailableElements,
averageNbElementForRemainingProc - nbElementsAlreadyOwned);
totalAvailableElements -= nbElementsToRecvPerProc[idxProc - firstProcToRecv];
totalRemainingElements -= nbElementsToRecvPerProc[idxProc - firstProcToRecv];
}
else{
......@@ -308,6 +312,10 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
counterValuesInPivot += 1;
}
}
if(counterValuesInPivot <= 1){
(*shouldStop) = true;
return globalPivot;
}
(*shouldStop) = false;
return globalPivot/counterValuesInPivot;
}
......@@ -336,7 +344,7 @@ public:
break;
}
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalPivot = " << globalPivot << "\n" );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalPivot = " << globalPivot << " for " << currentComm.processCount() << "\n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
// Split the array in two parts lower equal to pivot and greater than pivot
......
// ===================================================================================
// Copyright ScalFmm 2011 INRIA, Olivier Coulaud, Berenger Bramas
// olivier.coulaud@inria.fr, berenger.bramas@inria.fr
// This software is a computer program whose purpose is to compute the FMM.
//
// This software is governed by the CeCILL-C and LGPL licenses and
// abiding by the rules of distribution of free software.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public and CeCILL-C Licenses for more details.
// "http://www.cecill.info".
// "http://www.gnu.org/licenses".
// ===================================================================================
// ==== CMAKE =====
// @FUSE_MPI
// ================
#include <iostream>
#include <cstdio>
#include <cstdlib>
#include "../../Src/Kernels/Rotation/FRotationCell.hpp"
#include "../../Src/Kernels/Rotation/FRotationKernel.hpp"
#include "../../Src/Components/FSimpleLeaf.hpp"
#include "../../Src/Kernels/P2P/FP2PParticleContainerIndexed.hpp"
#include "../../Src/Utils/FParameters.hpp"
#include "../../Src/Utils/FMemUtils.hpp"
#include "../../Src/BalanceTree/FPartitionsMapping.hpp"
#include "../../Src/Containers/FOctree.hpp"
#include "../../Src/Containers/FVector.hpp"
#include "../../Src/Files/FRandomLoader.hpp"
#include "../../Src/Files/FMpiTreeBuilder.hpp"
#include "../../Src/Core/FFmmAlgorithm.hpp"
#include "../../Src/Core/FFmmAlgorithmThread.hpp"
#include "../../Src/Core/FFmmAlgorithmThreadProc.hpp"
#include "../../Src/BalanceTree/FLeafBalance.hpp"
#include "../../Src/Utils/FParameterNames.hpp"
/**
* This program runs the FMM Algorithm Distributed with the Rotation kernel
*/
// Simply create particles and try the kernels
int main(int argc, char* argv[])
{
FHelpDescribeAndExit(argc, argv,
"Test with MPI the chebyshev FMM and compare it to the direct computation for debugging purpose.",
FParameterDefinitions::NbParticles, FParameterDefinitions::OctreeHeight,
FParameterDefinitions::OctreeSubHeight, FParameterDefinitions::NbThreads);
typedef double FReal;
FMpi app(argc,argv);
const FSize nbParticles = FParameters::getValue(argc,argv, FParameterDefinitions::NbParticles.options, 10000000ULL);
const unsigned int TreeHeight = FParameters::getValue(argc, argv, FParameterDefinitions::OctreeHeight.options, 5);
FTic time;
std::cout << ">> This executable has to be used to test Proc Rotation Algorithm. \n";
// init particles position and physical value
struct TestParticle{
FPoint<FReal> position;
const FPoint<FReal>& getPosition(){
return position;
}
};
// open particle file
std::cout << "Creating : " << nbParticles << "\n" << std::endl;
FRandomLoader<FReal> loader(nbParticles, 1.0, FPoint<FReal>(0,0,0), app.global().processId());
time.tic();
std::unique_ptr<TestParticle[]> particles(new TestParticle[loader.getNumberOfParticles()]);
for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
loader.fillParticle(&particles[idxPart].position);
}
FPartitionsMapping<FReal> map(app.global());
FVector<FPartitionsMapping<double>::TestParticle<0> > finalParticles = map.distributeParticles<0>(loader.getNumberOfParticles(),
loader.getCenterOfBox(),
loader.getBoxWidth(), TreeHeight,
[&](const int idx, FPoint<FReal>* position, std::array<FReal, 0>* /*val*/){
position->setPosition(particles[idx].position.getX(),
particles[idx].position.getY(),
particles[idx].position.getZ());
});
// Test every particles exist
{
std::unique_ptr<int[]> count(new int[app.global().processCount() * nbParticles]);
memset(count.get(), 0, sizeof(int) * app.global().processCount() * nbParticles);
for(FSize part = 0 ; part < finalParticles.getSize() ; ++part){
const FSize idx = finalParticles[part].initialProcOwner*nbParticles + finalParticles[part].localIndex;
FAssertLF(count[idx] == 0)
count[idx] = 1;
}
FMpi::Assert( MPI_Allreduce(MPI_IN_PLACE, count.get(), app.global().processCount()*nbParticles, MPI_INT, MPI_SUM,
app.global().getComm()), __LINE__);
for(FSize part = 0 ; part < app.global().processCount()*nbParticles ; ++part){
FAssertLF(count[part] == 1);
}
}
// Test to send data
{
std::unique_ptr<std::array<FReal, 2>[]> toDistr(new std::array<FReal, 2>[nbParticles]);
for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
toDistr[idxPart][0] = app.global().processId();
toDistr[idxPart][1] = FReal(idxPart);
}
std::unique_ptr<std::array<FReal, 2>[]> res = map.distributeData<2>(toDistr.get());
for(FSize part = 0 ; part < finalParticles.getSize() ; ++part){
if(int(res[part][0]) != finalParticles[part].initialProcOwner)
std::cout << "[" << app.global().processId() << "]Res proc is " << res[part][0] << " should be " << finalParticles[part].initialProcOwner << std::endl;
if(int(res[part][1]) != finalParticles[part].localIndex)
std::cout << "[" << app.global().processId() << "]Res localidx is " << res[part][1] << " should be " << finalParticles[part].localIndex << std::endl;
}
std::unique_ptr<std::array<FReal, 2>[]> resBack = map.getResultingData<2>(res.get());
for(FSize part = 0 ; part < loader.getNumberOfParticles() ; ++part){
if(int(resBack[part][0]) != app.global().processId())
std::cout << "[" << app.global().processId() << "]ResBack proc is " << resBack[part][0] << " should be " << app.global().processId() << std::endl;
if(int(resBack[part][1]) != part)
std::cout << "[" << app.global().processId() << "]ResBack localidx is " << resBack[part][1] << " should be " << part << std::endl;
}
}
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment