From e5fb1841fd717734f8e15bc4f968f42996350c61 Mon Sep 17 00:00:00 2001 From: Martin Khannouz Date: Fri, 25 Mar 2016 15:26:26 +0100 Subject: [PATCH] Got something working on almoste all cases. The implicit part reproducing explicit mapping works well. But on the explicit side, it turn out that if all morton indexes ain't used, it segfault. Gotta investigate it. --- .../FGroupTaskStarpuImplicitAlgorithm.hpp | 29 +++++-------------- .../Core/FGroupTaskStarpuMpiAlgorithm.hpp | 14 +++++++++ .../testBlockedImplicitAlgorithm.cpp | 21 +++----------- Tests/GroupTree/testBlockedMpiAlgorithm.cpp | 17 ++--------- 4 files changed, 28 insertions(+), 53 deletions(-) diff --git a/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp b/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp index 3b4aefd3..047f4e5d 100644 --- a/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp +++ b/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp @@ -287,8 +287,7 @@ public: void syncData(){ for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size()))) { - if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel)) {//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel)) {//Clean only our data handle starpu_data_acquire(cellHandles[idxLevel][idxHandle].symb, STARPU_R); starpu_data_release(cellHandles[idxLevel][idxHandle].symb); starpu_data_acquire(cellHandles[idxLevel][idxHandle].up, STARPU_R); @@ -300,8 +299,7 @@ public: } { for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(particleHandles.size()))) { - if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1)) {//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1)) {//Clean only our data handle starpu_data_acquire(particleHandles[idxHandle].symb, STARPU_R); starpu_data_release(particleHandles[idxHandle].symb); starpu_data_acquire(particleHandles[idxHandle].down, STARPU_R); @@ -435,13 +433,6 @@ public: nodeRepartition[idxLevel][node_id][1] = nodeRepartition[idxLevel+1][node_id][1] >> 3; } } - //for(int idxLevel = 0; idxLevel < tree->getHeight(); ++idxLevel){ - //cout << "Level " << idxLevel << endl; - //for(int node_id = 0; node_id < nproc; ++node_id){ - //cout << "\t" << nodeRepartition[idxLevel][node_id][0] << " " << nodeRepartition[idxLevel][node_id][0] << endl; - //} - - //} } protected: /** @@ -842,8 +833,7 @@ protected: void cleanHandle(){ for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size())))//Clean only our data handle - if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel))//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel))//Clean only our data handle { starpu_data_unregister(cellHandles[idxLevel][idxHandle].symb); starpu_data_unregister(cellHandles[idxLevel][idxHandle].up); @@ -854,8 +844,7 @@ protected: } { for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(particleHandles.size()))) - if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1))//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1))//Clean only our data handle { starpu_data_unregister(particleHandles[idxHandle].symb); starpu_data_unregister(particleHandles[idxHandle].down); @@ -875,10 +864,8 @@ protected: cellHandles[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel)); for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){ const CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup); - int registeringNode = dataMappingBerenger(currentCells->getStartingIndex()+1, idxLevel); + int registeringNode = dataMappingBerenger(currentCells->getStartingIndex(), idxLevel); - //int registeringNode = dataMapping(idxGroup, tree->getNbCellGroupAtLevel(idxLevel)); - //cout << registeringNode << endl; where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1; starpu_variable_data_register(&cellHandles[idxLevel][idxGroup].symb, where, (uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte()); @@ -900,8 +887,7 @@ protected: { particleHandles.resize(tree->getNbParticleGroup()); for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){ - //int registeringNode = dataMapping(idxGroup, tree->getNbParticleGroup()); - int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex()+1, tree->getHeight()-1); + int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex(), tree->getHeight()-1); where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1; ParticleGroupClass* containers = tree->getParticleGroup(idxGroup); starpu_variable_data_register(&particleHandles[idxGroup].symb, where, @@ -924,12 +910,11 @@ protected: } } int dataMappingBerenger(MortonIndex const idx, int const idxLevel) const { - //return idxLevel%4; for(int i = 0; i < nproc; ++i) if(nodeRepartition[idxLevel][i][0] <= nodeRepartition[idxLevel][i][1] && idx >= nodeRepartition[idxLevel][i][0] && idx <= nodeRepartition[idxLevel][i][1]) return i; if(mpi_rank == 0) - cout << "Error !! [" << idx << "," << idxLevel << "]" << endl; + cout << "[scalfmm][map error] idx " << idx << " on level " << idxLevel << " isn't mapped on any proccess." << endl; return -1; } /** diff --git a/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp b/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp index 441cacc8..e0b224f3 100644 --- a/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp +++ b/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp @@ -317,8 +317,14 @@ public: ~FGroupTaskStarPUMpiAlgorithm(){ starpu_resume(); + std::cout << "Loutre " << comm.processId() << std::endl; + comm.barrier(); cleanHandle(); + std::cout << "Canard " << comm.processId() << std::endl; + comm.barrier(); cleanHandleMpi(); + std::cout << "Suricate " << comm.processId() << std::endl; + comm.barrier(); delete[] cellHandles; starpu_pthread_mutex_t releaseMutex; @@ -1344,12 +1350,16 @@ protected: for(int idxLevel = 0 ; idxLevel < int(remoteCellGroups.size()) ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(remoteCellGroups[idxLevel].size()) ; ++idxHandle){ if(remoteCellGroups[idxLevel][idxHandle].ptrSymb){ + if(comm.processId() == 4) + std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl; starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleSymb); starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleUp); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrSymb); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrUp); if(remoteCellGroups[idxLevel][idxHandle].ptrDown){ + if(comm.processId() == 4) + std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl; starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleDown); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrDown); } @@ -1360,12 +1370,16 @@ protected: { for(int idxHandle = 0 ; idxHandle < int(remoteParticleGroupss.size()) ; ++idxHandle){ if(remoteParticleGroupss[idxHandle].ptrSymb){ + if(comm.processId() == 4) + std::cout << "Narval part " << idxHandle << std::endl; starpu_data_unregister(remoteParticleGroupss[idxHandle].handleSymb); FAlignedMemory::DeallocBytes(remoteParticleGroupss[idxHandle].ptrSymb); } } remoteParticleGroupss.clear(); } + if(comm.processId() == 4) + std::cout << "Done" << std::endl; } //////////////////////////////////////////////////////////////////////////// diff --git a/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp b/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp index d08df067..5ecf3ea2 100644 --- a/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp +++ b/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp @@ -65,7 +65,7 @@ using namespace std; // FFmmAlgorithmTask FFmmAlgorithmThread typedef FFmmAlgorithm FmmClass; -//#define LOAD_FILE +#define LOAD_FILE #ifndef LOAD_FILE typedef FRandomLoader LoaderClass; #else @@ -100,8 +100,8 @@ int main(int argc, char* argv[]){ cout << "Pas de mpi -_-\" " << endl; #endif #ifndef LOAD_FILE - const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(100)); - LoaderClass loader(NbParticles, 1.0, FPoint(0,0,0), 0); + const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(10000)); + LoaderClass loader(NbParticles, 1.0, FPoint(0,0,0), 0); #else // Load the particles const char* const filename = FParameters::getStr(argc,argv,FParameterDefinitions::InputFile.options, "../Data/test20k.fma"); @@ -157,15 +157,6 @@ int main(int argc, char* argv[]){ mpi_rank = groupalgo.getRank(); cout << "Executing time (implicit node " << mpi_rank << ") " << elapsedTime << "s\n"; timeAverage(mpi_rank, nproc, elapsedTime); - //if( groupalgo.getRank() != 0) - //return 0; - //groupedTree.printInfoBlocks(); - //for(int i = 0; i < NbLevels; ++i) - //{ - //cout << "Level " << i << "(" << sizeForEachGroup[i].size() << ")" << endl; - //for(int j = 0; j < sizeForEachGroup[i].size(); ++j) - //cout << "\t" << sizeForEachGroup[i][j] << endl; - //} // Usual algorithm KernelClass kernels; // FTestKernels FBasicKernels @@ -276,7 +267,7 @@ void sortParticle(FPoint * allParticles, int treeHeight, int groupSize, v } //Compte le nombre de feuilles - sizeForEachGroup.resize(treeHeight+1);//Le +1 est pour les particules + sizeForEachGroup.resize(treeHeight); MortonIndex previousLeaf = -1; int numberOfLeaf = 0; for(FSize idxPart = 0 ; idxPart < nbParticles ; ++idxPart) @@ -331,10 +322,6 @@ void sortParticle(FPoint * allParticles, int treeHeight, int groupSize, v std::vector>> nodeRepartition; createNodeRepartition(distributedMortonIndex, nodeRepartition, nproc, treeHeight); - //for(int i = 0; i < nproc; ++i) - //{ - //cout << nodeRepartition[1][i][0] << " - " << nodeRepartition[1][i][1] << endl; - //} //Pour chaque niveau calcul de la taille des groupe for(int idxLevel = treeHeight - 2; idxLevel >= 0; --idxLevel) { diff --git a/Tests/GroupTree/testBlockedMpiAlgorithm.cpp b/Tests/GroupTree/testBlockedMpiAlgorithm.cpp index e28b1d2f..81d8f5b8 100644 --- a/Tests/GroupTree/testBlockedMpiAlgorithm.cpp +++ b/Tests/GroupTree/testBlockedMpiAlgorithm.cpp @@ -104,7 +104,7 @@ int main(int argc, char* argv[]){ memset(particles.get(), 0, sizeof(TestParticle) * loader.getNumberOfParticles()); for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){ loader.fillParticleAtMortonIndex(&(particles[idxPart].position), mpiComm.global().processId()*NbParticles + idxPart,NbLevels); - //loader.fillParticle(&(particles[idxPart].position)); + //loader.fillParticle(&(particles[idxPart].position)); } // Sort in parallel FVector myParticles; @@ -147,28 +147,20 @@ int main(int argc, char* argv[]){ std::cout << "Exchange particle to create the file" << std::endl; std::vector particlesGathered; std::vector sizeGathered; - std::vector mortonIndex; //Ajout des mes particules int sizeofParticle = sizeof(TestParticle)*myParticles.getSize(); sizeGathered.push_back(sizeofParticle); particlesGathered.push_back(new TestParticle[sizeofParticle]); memcpy(particlesGathered.back(), myParticles.data(), sizeofParticle); - mortonIndex.push_back(leftLimite); - mortonIndex.push_back(myLeftLimite); //Recupération des particules des autres for(int i = 1; i < mpiComm.global().processCount(); ++i) { int sizeReceive; - MortonIndex mortonStart, mortonEnd; MPI_Recv(&sizeReceive, sizeof(sizeReceive), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); sizeGathered.push_back(sizeReceive); particlesGathered.push_back(new TestParticle[sizeReceive]); MPI_Recv(particlesGathered.back(), sizeReceive, MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - MPI_Recv(&mortonStart, sizeof(mortonStart), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - MPI_Recv(&mortonEnd, sizeof(mortonEnd), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - mortonIndex.push_back(mortonStart); - mortonIndex.push_back(mortonEnd); } int sum = 0; for(int a : sizeGathered) @@ -180,10 +172,6 @@ int main(int argc, char* argv[]){ writer.writeHeader(loader.getCenterOfBox(), loader.getBoxWidth(),totalNbParticles, particles[0]); for(unsigned int i = 0; i < particlesGathered.size(); ++i) writer.writeArrayOfParticles(particlesGathered[i], sizeGathered[i]/sizeof(TestParticle)); - std::ofstream fichier("mapping", std::ios::out | std::ios::trunc); //déclaration du flux et ouverture du fichier - fichier << mortonIndex.size()/2 << std::endl; - for(unsigned int i = 0; i < mortonIndex.size(); i+=2) - fichier << mortonIndex[i] << " " << mortonIndex[i+1] << std::endl; for(TestParticle* ptr : particlesGathered) delete ptr; std::cout << "Done exchanging !" << std::endl; @@ -221,6 +209,7 @@ int main(int argc, char* argv[]){ }); + mpiComm.global().barrier(); typedef FTestCell CellClass; typedef FTestParticleContainer ContainerClass; @@ -236,7 +225,7 @@ int main(int argc, char* argv[]){ FRandomLoader loaderAll(NbParticles, 1.0, FPoint(0,0,0), idxProc); for(FSize idxPart = 0 ; idxPart < loaderAll.getNumberOfParticles() ; ++idxPart){ FPoint pos; - //loaderAll.fillParticle(&pos); + //loaderAll.fillParticle(&pos); loaderAll.fillParticleAtMortonIndex(&pos, idxProc*NbParticles + idxPart,NbLevels); tree.insert(pos); } -- GitLab