Commit 439e411b authored by BRAMAS Berenger's avatar BRAMAS Berenger
Browse files

update mpi classes

parent 70fc4abe
......@@ -108,9 +108,8 @@ public:
/** Write back, position + sizeof(object) has to be < size */
template <class ClassType>
void writeAt(const FSize position, const ClassType& object){
FAssertLF(position <= currentIndex);
currentIndex = position;
write(object);
FAssertLF(position+FSize(sizeof(ClassType)) <= currentIndex);
memcpy(&array[position], &object, sizeof(ClassType));
}
/** Write an array
......
......@@ -492,7 +492,7 @@ protected:
MPI_Isend(&bufferSize, 1, FMpi::GetType(bufferSize), currentProcIdToSendTo,
FMpi::TagFmmM2MSize + idxLevel, comm.getComm(), &requestsSize[iterMpiRequestsSize++]);
FAssertLF(sendBuffer.getSize() < std::numeric_limits<int>::max());
MPI_Isend(sendBuffer.data(), int(sendBuffer.getSize()), MPI_PACKED, currentProcIdToSendTo,
MPI_Isend(sendBuffer.data(), int(sendBuffer.getSize()), MPI_BYTE, currentProcIdToSendTo,
FMpi::TagFmmM2M + idxLevel, comm.getComm(), &requests[iterMpiRequests++]);
}
}
......@@ -533,7 +533,7 @@ protected:
if(procHasWorkAtLevel(idxLevel+1, idProcSource) && procCoversMyRightBorderCell(idxLevel, idProcSource)){
recvBuffer[nbProcThatSendToMe].cleanAndResize(recvBufferSize[nbProcThatSendToMe]);
FAssertLF(recvBufferSize[nbProcThatSendToMe] < std::numeric_limits<int>::max());
MPI_Irecv(recvBuffer[nbProcThatSendToMe].data(), int(recvBufferSize[nbProcThatSendToMe]), MPI_PACKED,
MPI_Irecv(recvBuffer[nbProcThatSendToMe].data(), int(recvBufferSize[nbProcThatSendToMe]), MPI_BYTE,
idProcSource, FMpi::TagFmmM2M + idxLevel, comm.getComm(), &requests[iterMpiRequests++]);
nbProcThatSendToMe += 1;
FAssertLF(nbProcThatSendToMe <= 7);
......@@ -557,7 +557,7 @@ protected:
// Retreive data and merge my child and the child from others
for(int idxProc = 0 ; idxProc < nbProcThatSendToMe ; ++idxProc){
int packageFlags = int(recvBuffer[idxProc].getValue<char>());
unsigned packageFlags = unsigned(recvBuffer[idxProc].getValue<unsigned char>());
int position = 0;
int positionToInsert = 0;
......@@ -1113,7 +1113,7 @@ protected:
FMpi::MpiAssert( MPI_Isend(&sendBufferSize, 1, FMpi::GetType(sendBufferSize), idxProcSend,
FMpi::TagFmmL2LSize + idxLevel, comm.getComm(), &requestsSize[iterRequestsSize++]), __LINE__);
FAssertLF(sendBuffer.getSize() < std::numeric_limits<int>::max());
FMpi::MpiAssert( MPI_Isend(sendBuffer.data(), int(sendBuffer.getSize()), MPI_PACKED, idxProcSend,
FMpi::MpiAssert( MPI_Isend(sendBuffer.data(), int(sendBuffer.getSize()), MPI_BYTE, idxProcSend,
FMpi::TagFmmL2L + idxLevel, comm.getComm(), &requests[iterRequests++]), __LINE__);
// Inc and check the counter
nbMessageSent += 1;
......@@ -1136,7 +1136,7 @@ protected:
if(hasToReceive){
recvBuffer.cleanAndResize(recvBufferSize);
FAssertLF(recvBuffer.getCapacity() < std::numeric_limits<int>::max());
FMpi::MpiAssert( MPI_Irecv( recvBuffer.data(), int(recvBuffer.getCapacity()), MPI_PACKED, idxProcToReceive,
FMpi::MpiAssert( MPI_Irecv( recvBuffer.data(), int(recvBuffer.getCapacity()), MPI_BYTE, idxProcToReceive,
FMpi::TagFmmL2L + idxLevel, comm.getComm(), &requests[iterRequests++]), __LINE__ );
}
......
......@@ -20,6 +20,7 @@
#include "../Utils/FQuickSortMpi.hpp"
#include "../Utils/FBitonicSort.hpp"
#include "../Utils/FTic.hpp"
#include "../Utils/FEnv.hpp"
#include "../Utils/FMemUtils.hpp"
......@@ -41,6 +42,8 @@
template<class FReal, class ParticleClass>
class FMpiTreeBuilder{
private:
static const bool VerboseLog;
/** To keep the leaves information after the sort */
struct LeafInfo {
MortonIndex mindex;
......@@ -188,8 +191,8 @@ public:
if( (*workingSize) != 0 ){
borderLeavesState[0] = leavesInfo[0];
borderLeavesState[1] = leavesInfo[leavesInfo.getSize()-1];
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] First " << borderLeavesState[0].mindex << "\n"; FLog::Controller.flush(); );
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Last " << borderLeavesState[1].mindex << "\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] First " << borderLeavesState[0].mindex << "\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Last " << borderLeavesState[1].mindex << "\n"; FLog::Controller.flush(); );
}
std::unique_ptr<LeafInfo[]> allProcFirstLeafStates(new LeafInfo[nbProcs*2]);
......@@ -213,7 +216,7 @@ public:
// Post and send message for the first leaf
FMpi::ISendSplit(&workingArray[0], borderLeavesState[0].nbParts, idProcToSendTo,
FMpi::TagExchangeIndexs, communicator, &requests);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] send " << borderLeavesState[0].nbParts << " to " << idProcToSendTo << "\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] send " << borderLeavesState[0].nbParts << " to " << idProcToSendTo << "\n"; FLog::Controller.flush(); );
hasSentFirstLeaf = true;
}
}
......@@ -242,7 +245,7 @@ public:
if(allProcFirstLeafStates[(postRecvIdx)*2].mindex != noDataFlag){
FMpi::IRecvSplit(&receivedParticles[postPositionRecv], allProcFirstLeafStates[(postRecvIdx)*2].nbParts, postRecvIdx,
FMpi::TagExchangeIndexs, communicator, &requests);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv " << allProcFirstLeafStates[(postRecvIdx)*2].nbParts << " from " << postRecvIdx << "\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv " << allProcFirstLeafStates[(postRecvIdx)*2].nbParts << " from " << postRecvIdx << "\n"; FLog::Controller.flush(); );
// Inc the write position
postPositionRecv += allProcFirstLeafStates[(postRecvIdx)*2].nbParts;
}
......@@ -336,7 +339,7 @@ public:
1, MPI_LONG_LONG_INT, communicator.getComm()), __LINE__);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Exchange number of leaves\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Exchange number of leaves\n"; FLog::Controller.flush(); );
// prefix sum
std::unique_ptr<FSize[]> diffNumberOfLeavesPerProc(new FSize[nbProcs+1]);
......@@ -358,8 +361,8 @@ public:
std::pair<size_t, size_t> myCurrentInter = {diffNumberOfLeavesPerProc[myRank], diffNumberOfLeavesPerProc[myRank+1]};
const std::vector<FEqualize::Package> packsToSend = FEqualize::GetPackToSend(myCurrentInter, allObjectives);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Get my interval (" << packsToSend.size() << ")\n"; FLog::Controller.flush(); );
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send data\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Get my interval (" << packsToSend.size() << ")\n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send data\n"; FLog::Controller.flush(); );
// Store the requests
std::vector<MPI_Request> requestsNbParts;
......@@ -371,17 +374,17 @@ public:
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
// If not to me and if there is something to send
const long long int nbPartsPerPackToSend = leavesOffsetInParticles[pack.elementTo]-leavesOffsetInParticles[pack.elementFrom];
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-send to " << pack.idProc << " nb " << nbPartsPerPackToSend << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-send to " << pack.idProc << " nb " << nbPartsPerPackToSend << " \n"; FLog::Controller.flush(); );
// Send the size of the data
requestsNbParts.emplace_back();
FMpi::MpiAssert(MPI_Isend(&nbPartsPerPackToSend,1,MPI_LONG_LONG_INT,pack.idProc,
FMpi::TagExchangeIndexs, communicator.getComm(), &requestsNbParts.back()),__LINE__);
}
else {
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
}
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send done \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Send done \n"; FLog::Controller.flush(); );
// Compute the current intervals
std::vector< std::pair<size_t,size_t> > allCurrentIntervals;
......@@ -391,17 +394,17 @@ public:
allCurrentIntervals[idxProc].second = diffNumberOfLeavesPerProc[idxProc+1];
}
// Ask the packs to receive to fill my objective
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Get my receive interval \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Get my receive interval \n"; FLog::Controller.flush(); );
std::pair<size_t, size_t> myObjective = allObjectives[myRank];
const std::vector<FEqualize::Package> packsToRecv = FEqualize::GetPackToRecv(myObjective, allCurrentIntervals);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv nb particles \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv nb particles \n"; FLog::Controller.flush(); );
// Count the number of parts to receive
std::unique_ptr<FSize[]> nbPartsPerPackToRecv(new FSize[packsToRecv.size()]);
for(unsigned int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-recv from " << pack.idProc << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre-recv from " << pack.idProc << " \n"; FLog::Controller.flush(); );
// We need to know how much particles to receive
requestsNbParts.emplace_back();
FMpi::MpiAssert(MPI_Irecv(&nbPartsPerPackToRecv[idxPack], 1, MPI_LONG_LONG_INT, pack.idProc,
......@@ -409,7 +412,7 @@ public:
}
else{
if(pack.idProc == myRank){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip recv " << idxPack << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip recv " << idxPack << " \n"; FLog::Controller.flush(); );
// Take my own data
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
const FSize nbLeavesToCopy = pack.elementTo-pack.elementFrom;
......@@ -422,18 +425,18 @@ public:
}
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait \n"; FLog::Controller.flush(); );
FMpi::MpiAssert(MPI_Waitall(int(requestsNbParts.size()), requestsNbParts.data(), MPI_STATUSES_IGNORE), __LINE__);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait Done \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait Done \n"; FLog::Controller.flush(); );
std::vector<MPI_Request> requestsParts;
for(unsigned int idxPack = 0; idxPack< packsToSend.size() ; ++idxPack){
const FEqualize::Package& pack = packsToSend[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] send to "
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] send to "
<< pack.idProc << " nb " << (pack.elementTo-pack.elementFrom) << " \n"; FLog::Controller.flush(); );
FMpi::ISendSplit(&particlesArrayInLeafOrder[pack.elementFrom],
......@@ -446,7 +449,7 @@ public:
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] barrier after all send \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] barrier after all send \n"; FLog::Controller.flush(); );
// Count the number of leaf to receive
FSize totalPartsToReceive = 0;
......@@ -462,7 +465,7 @@ public:
for(unsigned int idxPack = 0; idxPack < packsToRecv.size(); ++idxPack){
const FEqualize::Package& pack = packsToRecv[idxPack];
if(pack.idProc != myRank && 0 < (pack.elementTo-pack.elementFrom)){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv from "
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] recv from "
<< pack.idProc << " nb " << nbPartsPerPackToRecv[idxPack] << "\n"; FLog::Controller.flush(); );
FAssertLF(pack.elementTo <= size_t(totalPartsToReceive));
......@@ -475,7 +478,7 @@ public:
}
else if(pack.idProc == myRank){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] skip " << idxPack << " \n"; FLog::Controller.flush(); );
// Copy my particles
const FSize sourcePosition = FMath::Max(myObjective.first, myCurrentInter.first) - myCurrentInter.first;
memcpy(&particlesRecvBuffer[offsetToRecv], &particlesArrayInLeafOrder[leavesOffsetInParticles[sourcePosition]],
......@@ -486,11 +489,11 @@ public:
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre Wait \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] pre Wait \n"; FLog::Controller.flush(); );
FMpi::Assert( MPI_Waitall(int(requestsParts.size()), requestsParts.data(), MPI_STATUSES_IGNORE), __LINE__ );
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait Done \n"; FLog::Controller.flush(); );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << communicator.processId() << "] Wait Done \n"; FLog::Controller.flush(); );
// Insert in the particle saver
for(FSize idPartsToStore = 0 ; idPartsToStore < totalPartsToReceive ; ++idPartsToStore){
......@@ -576,4 +579,10 @@ public:
};
#ifdef SCALFMM_USE_LOG
template<class FReal, class ParticleClass>
const bool FMpiTreeBuilder<FReal,ParticleClass>::VerboseLog = FEnv::GetBool("SCALFMM_DEBUG_LOG", false);
#endif
#endif // FMPITREEBUILDER_H
......@@ -20,12 +20,17 @@
#include "FMpi.hpp"
#include "FLog.hpp"
#include "FAssert.hpp"
#include "FEnv.hpp"
#include <memory>
#include <utility>
template <class SortType, class CompareType, class IndexType = size_t>
class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
#ifdef SCALFMM_USE_LOG
static const bool VerboseLog;
#endif
// We need a structure see the algorithm detail to know more
struct Partition{
IndexType lowerPart;
......@@ -102,7 +107,7 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
FAssert((averageNbElementForRemainingProc - nbElementsAlreadyOwned) == 0);
nbElementsToRecvPerProc[idxProc - firstProcToRecv] = 0;
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentRank << "] nbElementsToRecvPerProc[" << idxProc << "] = " << nbElementsToRecvPerProc[idxProc - firstProcToRecv] << "\n"; )
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentRank << "] nbElementsToRecvPerProc[" << idxProc << "] = " << nbElementsToRecvPerProc[idxProc - firstProcToRecv] << "\n"; )
}
FAssertLF(totalRemainingElements == 0);
}
......@@ -113,7 +118,7 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
for(int idxProc = firstProcToSend; idxProc < lastProcToSend ; ++idxProc){
const IndexType nbElementsAlreadyOwned = (inFromRightToLeft ? globalElementBalance[idxProc].lowerPart : globalElementBalance[idxProc].greaterPart);
nbElementsToSendPerProc[idxProc-firstProcToSend] = nbElementsAlreadyOwned;
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentRank << "] nbElementsToSendPerProc[" << idxProc << "] = " << nbElementsToSendPerProc[idxProc-firstProcToSend] << "\n"; )
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentRank << "] nbElementsToSendPerProc[" << idxProc << "] = " << nbElementsToSendPerProc[idxProc-firstProcToSend] << "\n"; )
}
// Compute all the send recv but keep only the ones related to currentRank
......@@ -177,23 +182,23 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
requests.reserve(whatToRecvFromWho.size());
for(int idxPack = 0 ; idxPack < int(whatToRecvFromWho.size()) ; ++idxPack){
const PackData& pack = whatToRecvFromWho[idxPack];
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Recv from " << pack.idProc << " from " << pack.fromElement << " to " << pack.toElement << "\n"; );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Recv from " << pack.idProc << " from " << pack.fromElement << " to " << pack.toElement << "\n"; );
FAssertLF(pack.toElement <= totalToRecv);
FMpi::IRecvSplit(&recvBuffer[pack.fromElement],
(pack.toElement - pack.fromElement),
pack.idProc,
FMpi::TagQuickSort,
currentComm,
&requests);
(pack.toElement - pack.fromElement),
pack.idProc,
FMpi::TagQuickSort,
currentComm,
&requests);
}
FAssertLF(whatToRecvFromWho.size() <= requests.size());
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << "Wait for " << requests.size() << " request \n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << "Wait for " << requests.size() << " request \n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
// Wait to complete
FMpi::Assert( MPI_Waitall(int(requests.size()), requests.data(), MPI_STATUSES_IGNORE), __LINE__ );
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Recv Done \n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Recv Done \n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
// Copy to ouput variables
(*inPartRecv) = recvBuffer;
(*inNbElementsRecv) = totalToRecv;
......@@ -212,22 +217,22 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
requests.reserve(whatToSendToWho.size());
for(int idxPack = 0 ; idxPack < int(whatToSendToWho.size()) ; ++idxPack){
const PackData& pack = whatToSendToWho[idxPack];
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Send to " << pack.idProc << " from " << pack.fromElement << " to " << pack.toElement << "\n"; );
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Send to " << pack.idProc << " from " << pack.fromElement << " to " << pack.toElement << "\n"; );
FMpi::ISendSplit(&inPartToSend[pack.fromElement],
(pack.toElement - pack.fromElement),
pack.idProc,
FMpi::TagQuickSort,
currentComm,
&requests);
(pack.toElement - pack.fromElement),
pack.idProc,
FMpi::TagQuickSort,
currentComm,
&requests);
}
FAssertLF(whatToSendToWho.size() <= requests.size());
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Wait for " << requests.size() << " request \n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Wait for " << requests.size() << " request \n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
// Wait to complete
FMpi::Assert( MPI_Waitall(int(requests.size()), requests.data(), MPI_STATUSES_IGNORE), __LINE__ );
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Send Done \n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Send Done \n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
}
static CompareType SelectPivot(const SortType workingArray[], const IndexType currentSize, const FMpi::FComm& currentComm, bool* shouldStop){
......@@ -260,14 +265,14 @@ class FQuickSortMpi : public FQuickSort< SortType, IndexType> {
localPivot = ((maxFoundValue-minFoundValue)/2) + minFoundValue;
// The pivot must be different (to ensure that the partition will return two parts)
if( localPivot == maxFoundValue && !allTheSame){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Pivot " << localPivot << " is equal max and allTheSame equal " << allTheSame << "\n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Pivot " << localPivot << " is equal max and allTheSame equal " << allTheSame << "\n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
localPivot -= 1;
}
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] localPivot = " << localPivot << "\n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] localPivot = " << localPivot << "\n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
//const int myRank = currentComm.processId();
const int nbProcs = currentComm.processCount();
......@@ -327,20 +332,20 @@ public:
bool shouldStop;
const CompareType globalPivot = SelectPivot(workingArray, currentSize, currentComm, &shouldStop);
if(shouldStop){
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] shouldStop = " << shouldStop << "\n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] shouldStop = " << shouldStop << "\n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
break;
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalPivot = " << globalPivot << "\n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalPivot = " << globalPivot << "\n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
// Split the array in two parts lower equal to pivot and greater than pivot
const IndexType nbLowerElements = QsPartition(workingArray, 0, currentSize-1, globalPivot);
const IndexType nbGreaterElements = currentSize - nbLowerElements;
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] After Partition: lower = " << nbLowerElements << " greater = " << nbGreaterElements << "\n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] After Partition: lower = " << nbLowerElements << " greater = " << nbGreaterElements << "\n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
const int currentRank = currentComm.processId();
const int currentNbProcs = currentComm.processCount();
......@@ -366,19 +371,19 @@ public:
globalNumberOfElementsLower += globalElementBalance[idxProc].lowerPart;
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalNumberOfElementsGreater = " << globalNumberOfElementsGreater << "\n"; )
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalNumberOfElementsLower = " << globalNumberOfElementsLower << "\n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalNumberOfElementsGreater = " << globalNumberOfElementsGreater << "\n"; )
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] globalNumberOfElementsLower = " << globalNumberOfElementsLower << "\n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
// The proc rank in the middle from the percentage
int procInTheMiddle;
if(globalNumberOfElementsLower == 0) procInTheMiddle = -1;
else if(globalNumberOfElementsGreater == 0) procInTheMiddle = currentNbProcs-1;
else procInTheMiddle = int(FMath::Min(IndexType(currentNbProcs-2), (currentNbProcs*globalNumberOfElementsLower)
/(globalNumberOfElementsGreater + globalNumberOfElementsLower)));
/(globalNumberOfElementsGreater + globalNumberOfElementsLower)));
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] procInTheMiddle = " << procInTheMiddle << "\n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] procInTheMiddle = " << procInTheMiddle << "\n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
// Send or receive depending on the state
if(currentRank <= procInTheMiddle){
......@@ -399,11 +404,11 @@ public:
workingArray = fullLowerPart;
currentSize = fullNbLowerElementsRecv;
// Reduce working group
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Reduce group to " << 0 << " / " << procInTheMiddle << "\n"; )
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Reduce group to " << 0 << " / " << procInTheMiddle << "\n"; )
FLOG(if(VerboseLog) FLog::Controller.flush());
currentComm.groupReduce( 0, procInTheMiddle);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Done\n" );
FLOG( FLog::Controller.flush());
FLOG(if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Done\n" );
FLOG(if(VerboseLog) FLog::Controller.flush());
}
else {
// I am in the group of the greater elements
......@@ -423,16 +428,16 @@ public:
workingArray = fullGreaterPart;
currentSize = fullNbGreaterElementsRecv;
// Reduce working group
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Reduce group to " << procInTheMiddle + 1 << " / " << currentNbProcs - 1 << "\n"; )
FLOG( FLog::Controller.flush());
FLOG( if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Reduce group to " << procInTheMiddle + 1 << " / " << currentNbProcs - 1 << "\n"; )
FLOG( if(VerboseLog) FLog::Controller.flush());
currentComm.groupReduce( procInTheMiddle + 1, currentNbProcs - 1);
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Done\n"; )
FLOG( FLog::Controller.flush());
FLOG( if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Done\n"; )
FLOG( if(VerboseLog) FLog::Controller.flush());
}
}
FLOG( FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Sequential sort (currentSize = " << currentSize << ")\n"; )
FLOG( FLog::Controller.flush());
FLOG( if(VerboseLog) FLog::Controller << "SCALFMM-DEBUG [" << currentComm.processId() << "] Sequential sort (currentSize = " << currentSize << ")\n"; )
FLOG( if(VerboseLog) FLog::Controller.flush());
// Finish by a local sort
FQuickSort< SortType, IndexType>::QsOmp(workingArray, currentSize, [](const SortType& v1, const SortType& v2){
return CompareType(v1) <= CompareType(v2);
......@@ -442,4 +447,10 @@ public:
}
};
#ifdef SCALFMM_USE_LOG
template <class SortType, class CompareType, class IndexType>
const bool FQuickSortMpi<SortType, CompareType, IndexType>::VerboseLog = FEnv::GetBool("SCALFMM_DEBUG_LOG", false);
#endif
#endif // FQUICKSORTMPI_HPP
......@@ -115,8 +115,8 @@ int main(int argc, char* argv[])
tree.getBoxWidth(),tree.getHeight(),
&finalParticles, &balancer);
{ // -----------------------------------------------------
std::cout << "Creating & Inserting " << finalParticles.getSize() << " particles ..." << std::endl;
std::cout << "For a total of " << loader.getNumberOfParticles() * app.global().processCount() << " particles ..." << std::endl;
std::cout << app.global().processId() << "] Creating & Inserting " << finalParticles.getSize() << " particles ..." << std::endl;
std::cout << app.global().processId() << "] For a total of " << loader.getNumberOfParticles() * app.global().processCount() << " particles ..." << std::endl;
std::cout << "\tHeight : " << TreeHeight << " \t sub-height : " << SubTreeHeight << std::endl;
time.tic();
......@@ -126,8 +126,17 @@ int main(int argc, char* argv[])
}
time.tac();
std::cout << "Done " << "(@Creating and Inserting Particles = "
std::cout << app.global().processId() << "] Done " << "(@Creating and Inserting Particles = "
<< time.elapsed() << "s)." << std::endl;
FSize minPart = std::numeric_limits<FSize>::max();
FSize maxPart = std::numeric_limits<FSize>::min();
tree.forEachLeaf([&](LeafClass* lf){
minPart = FMath::Min(lf->getSrc()->getNbParticles(), minPart);
maxPart = FMath::Max(lf->getSrc()->getNbParticles(), maxPart);
});
std::cout << app.global().processId() << "] Min nb part " << minPart << " Max nb part " << maxPart << std::endl;
} // -----------------------------------------------------
delete[] particles;
......@@ -139,13 +148,14 @@ int main(int argc, char* argv[])
KernelClass kernels(TreeHeight, loader.getBoxWidth(), loader.getCenterOfBox());
FmmClass algorithm(app.global(),&tree, &kernels);
time.tac();
std::cout << "Done " << "(@Init = " << time.elapsed() << "s)." << std::endl;
std::cout << app.global().processId() << "] Done " << "(@Init = " << time.elapsed() << "s)." << std::endl;
time.tic();
algorithm.execute();
time.tac();
std::cout << "Done " << "(@Algorithm = " << time.elapsed() << "s)." << std::endl;
std::cout << app.global().processId() << "] Done " << "(@Algorithm = " << time.elapsed() << "s)." << std::endl;
} // -----------------------------------------------------
app.global().barrier();
return 0;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment