diff --git a/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp b/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp index 3b4aefd33d7645d715d850c57accc76af85b0b9e..047f4e5d04080f737f44bc07e98840b1d025adf1 100644 --- a/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp +++ b/Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp @@ -287,8 +287,7 @@ public: void syncData(){ for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size()))) { - if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel)) {//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel)) {//Clean only our data handle starpu_data_acquire(cellHandles[idxLevel][idxHandle].symb, STARPU_R); starpu_data_release(cellHandles[idxLevel][idxHandle].symb); starpu_data_acquire(cellHandles[idxLevel][idxHandle].up, STARPU_R); @@ -300,8 +299,7 @@ public: } { for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(particleHandles.size()))) { - if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1)) {//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1)) {//Clean only our data handle starpu_data_acquire(particleHandles[idxHandle].symb, STARPU_R); starpu_data_release(particleHandles[idxHandle].symb); starpu_data_acquire(particleHandles[idxHandle].down, STARPU_R); @@ -435,13 +433,6 @@ public: nodeRepartition[idxLevel][node_id][1] = nodeRepartition[idxLevel+1][node_id][1] >> 3; } } - //for(int idxLevel = 0; idxLevel < tree->getHeight(); ++idxLevel){ - //cout << "Level " << idxLevel << endl; - //for(int node_id = 0; node_id < nproc; ++node_id){ - //cout << "\t" << nodeRepartition[idxLevel][node_id][0] << " " << nodeRepartition[idxLevel][node_id][0] << endl; - //} - - //} } protected: /** @@ -842,8 +833,7 @@ protected: void cleanHandle(){ for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size())))//Clean only our data handle - if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel))//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel))//Clean only our data handle { starpu_data_unregister(cellHandles[idxLevel][idxHandle].symb); starpu_data_unregister(cellHandles[idxLevel][idxHandle].up); @@ -854,8 +844,7 @@ protected: } { for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){ - //if(isDataOwned(idxHandle, int(particleHandles.size()))) - if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1))//Clean only our data handle + if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1))//Clean only our data handle { starpu_data_unregister(particleHandles[idxHandle].symb); starpu_data_unregister(particleHandles[idxHandle].down); @@ -875,10 +864,8 @@ protected: cellHandles[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel)); for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){ const CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup); - int registeringNode = dataMappingBerenger(currentCells->getStartingIndex()+1, idxLevel); + int registeringNode = dataMappingBerenger(currentCells->getStartingIndex(), idxLevel); - //int registeringNode = dataMapping(idxGroup, tree->getNbCellGroupAtLevel(idxLevel)); - //cout << registeringNode << endl; where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1; starpu_variable_data_register(&cellHandles[idxLevel][idxGroup].symb, where, (uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte()); @@ -900,8 +887,7 @@ protected: { particleHandles.resize(tree->getNbParticleGroup()); for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){ - //int registeringNode = dataMapping(idxGroup, tree->getNbParticleGroup()); - int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex()+1, tree->getHeight()-1); + int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex(), tree->getHeight()-1); where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1; ParticleGroupClass* containers = tree->getParticleGroup(idxGroup); starpu_variable_data_register(&particleHandles[idxGroup].symb, where, @@ -924,12 +910,11 @@ protected: } } int dataMappingBerenger(MortonIndex const idx, int const idxLevel) const { - //return idxLevel%4; for(int i = 0; i < nproc; ++i) if(nodeRepartition[idxLevel][i][0] <= nodeRepartition[idxLevel][i][1] && idx >= nodeRepartition[idxLevel][i][0] && idx <= nodeRepartition[idxLevel][i][1]) return i; if(mpi_rank == 0) - cout << "Error !! [" << idx << "," << idxLevel << "]" << endl; + cout << "[scalfmm][map error] idx " << idx << " on level " << idxLevel << " isn't mapped on any proccess." << endl; return -1; } /** diff --git a/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp b/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp index 441cacc812cd7b7368bfaaac161b64f4b2c18307..e0b224f39b57da85baf4d2f0cf10888d1413f460 100644 --- a/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp +++ b/Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp @@ -317,8 +317,14 @@ public: ~FGroupTaskStarPUMpiAlgorithm(){ starpu_resume(); + std::cout << "Loutre " << comm.processId() << std::endl; + comm.barrier(); cleanHandle(); + std::cout << "Canard " << comm.processId() << std::endl; + comm.barrier(); cleanHandleMpi(); + std::cout << "Suricate " << comm.processId() << std::endl; + comm.barrier(); delete[] cellHandles; starpu_pthread_mutex_t releaseMutex; @@ -1344,12 +1350,16 @@ protected: for(int idxLevel = 0 ; idxLevel < int(remoteCellGroups.size()) ; ++idxLevel){ for(int idxHandle = 0 ; idxHandle < int(remoteCellGroups[idxLevel].size()) ; ++idxHandle){ if(remoteCellGroups[idxLevel][idxHandle].ptrSymb){ + if(comm.processId() == 4) + std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl; starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleSymb); starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleUp); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrSymb); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrUp); if(remoteCellGroups[idxLevel][idxHandle].ptrDown){ + if(comm.processId() == 4) + std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl; starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleDown); FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrDown); } @@ -1360,12 +1370,16 @@ protected: { for(int idxHandle = 0 ; idxHandle < int(remoteParticleGroupss.size()) ; ++idxHandle){ if(remoteParticleGroupss[idxHandle].ptrSymb){ + if(comm.processId() == 4) + std::cout << "Narval part " << idxHandle << std::endl; starpu_data_unregister(remoteParticleGroupss[idxHandle].handleSymb); FAlignedMemory::DeallocBytes(remoteParticleGroupss[idxHandle].ptrSymb); } } remoteParticleGroupss.clear(); } + if(comm.processId() == 4) + std::cout << "Done" << std::endl; } //////////////////////////////////////////////////////////////////////////// diff --git a/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp b/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp index d08df067d01083aafc9bb78cea444e398ab5f6ae..5ecf3ea229b4d04bc8dc512a3e0387ba64e3ce9d 100644 --- a/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp +++ b/Tests/GroupTree/testBlockedImplicitAlgorithm.cpp @@ -65,7 +65,7 @@ using namespace std; // FFmmAlgorithmTask FFmmAlgorithmThread typedef FFmmAlgorithm FmmClass; -//#define LOAD_FILE +#define LOAD_FILE #ifndef LOAD_FILE typedef FRandomLoader LoaderClass; #else @@ -100,8 +100,8 @@ int main(int argc, char* argv[]){ cout << "Pas de mpi -_-\" " << endl; #endif #ifndef LOAD_FILE - const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(100)); - LoaderClass loader(NbParticles, 1.0, FPoint(0,0,0), 0); + const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(10000)); + LoaderClass loader(NbParticles, 1.0, FPoint(0,0,0), 0); #else // Load the particles const char* const filename = FParameters::getStr(argc,argv,FParameterDefinitions::InputFile.options, "../Data/test20k.fma"); @@ -157,15 +157,6 @@ int main(int argc, char* argv[]){ mpi_rank = groupalgo.getRank(); cout << "Executing time (implicit node " << mpi_rank << ") " << elapsedTime << "s\n"; timeAverage(mpi_rank, nproc, elapsedTime); - //if( groupalgo.getRank() != 0) - //return 0; - //groupedTree.printInfoBlocks(); - //for(int i = 0; i < NbLevels; ++i) - //{ - //cout << "Level " << i << "(" << sizeForEachGroup[i].size() << ")" << endl; - //for(int j = 0; j < sizeForEachGroup[i].size(); ++j) - //cout << "\t" << sizeForEachGroup[i][j] << endl; - //} // Usual algorithm KernelClass kernels; // FTestKernels FBasicKernels @@ -276,7 +267,7 @@ void sortParticle(FPoint * allParticles, int treeHeight, int groupSize, v } //Compte le nombre de feuilles - sizeForEachGroup.resize(treeHeight+1);//Le +1 est pour les particules + sizeForEachGroup.resize(treeHeight); MortonIndex previousLeaf = -1; int numberOfLeaf = 0; for(FSize idxPart = 0 ; idxPart < nbParticles ; ++idxPart) @@ -331,10 +322,6 @@ void sortParticle(FPoint * allParticles, int treeHeight, int groupSize, v std::vector>> nodeRepartition; createNodeRepartition(distributedMortonIndex, nodeRepartition, nproc, treeHeight); - //for(int i = 0; i < nproc; ++i) - //{ - //cout << nodeRepartition[1][i][0] << " - " << nodeRepartition[1][i][1] << endl; - //} //Pour chaque niveau calcul de la taille des groupe for(int idxLevel = treeHeight - 2; idxLevel >= 0; --idxLevel) { diff --git a/Tests/GroupTree/testBlockedMpiAlgorithm.cpp b/Tests/GroupTree/testBlockedMpiAlgorithm.cpp index e28b1d2f92e9d984d74d1bd2aad35f5e053504ba..81d8f5b875d9daab52fa99ac48e425281c27fe61 100644 --- a/Tests/GroupTree/testBlockedMpiAlgorithm.cpp +++ b/Tests/GroupTree/testBlockedMpiAlgorithm.cpp @@ -104,7 +104,7 @@ int main(int argc, char* argv[]){ memset(particles.get(), 0, sizeof(TestParticle) * loader.getNumberOfParticles()); for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){ loader.fillParticleAtMortonIndex(&(particles[idxPart].position), mpiComm.global().processId()*NbParticles + idxPart,NbLevels); - //loader.fillParticle(&(particles[idxPart].position)); + //loader.fillParticle(&(particles[idxPart].position)); } // Sort in parallel FVector myParticles; @@ -147,28 +147,20 @@ int main(int argc, char* argv[]){ std::cout << "Exchange particle to create the file" << std::endl; std::vector particlesGathered; std::vector sizeGathered; - std::vector mortonIndex; //Ajout des mes particules int sizeofParticle = sizeof(TestParticle)*myParticles.getSize(); sizeGathered.push_back(sizeofParticle); particlesGathered.push_back(new TestParticle[sizeofParticle]); memcpy(particlesGathered.back(), myParticles.data(), sizeofParticle); - mortonIndex.push_back(leftLimite); - mortonIndex.push_back(myLeftLimite); //Recupération des particules des autres for(int i = 1; i < mpiComm.global().processCount(); ++i) { int sizeReceive; - MortonIndex mortonStart, mortonEnd; MPI_Recv(&sizeReceive, sizeof(sizeReceive), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); sizeGathered.push_back(sizeReceive); particlesGathered.push_back(new TestParticle[sizeReceive]); MPI_Recv(particlesGathered.back(), sizeReceive, MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - MPI_Recv(&mortonStart, sizeof(mortonStart), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - MPI_Recv(&mortonEnd, sizeof(mortonEnd), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE); - mortonIndex.push_back(mortonStart); - mortonIndex.push_back(mortonEnd); } int sum = 0; for(int a : sizeGathered) @@ -180,10 +172,6 @@ int main(int argc, char* argv[]){ writer.writeHeader(loader.getCenterOfBox(), loader.getBoxWidth(),totalNbParticles, particles[0]); for(unsigned int i = 0; i < particlesGathered.size(); ++i) writer.writeArrayOfParticles(particlesGathered[i], sizeGathered[i]/sizeof(TestParticle)); - std::ofstream fichier("mapping", std::ios::out | std::ios::trunc); //déclaration du flux et ouverture du fichier - fichier << mortonIndex.size()/2 << std::endl; - for(unsigned int i = 0; i < mortonIndex.size(); i+=2) - fichier << mortonIndex[i] << " " << mortonIndex[i+1] << std::endl; for(TestParticle* ptr : particlesGathered) delete ptr; std::cout << "Done exchanging !" << std::endl; @@ -221,6 +209,7 @@ int main(int argc, char* argv[]){ }); + mpiComm.global().barrier(); typedef FTestCell CellClass; typedef FTestParticleContainer ContainerClass; @@ -236,7 +225,7 @@ int main(int argc, char* argv[]){ FRandomLoader loaderAll(NbParticles, 1.0, FPoint(0,0,0), idxProc); for(FSize idxPart = 0 ; idxPart < loaderAll.getNumberOfParticles() ; ++idxPart){ FPoint pos; - //loaderAll.fillParticle(&pos); + //loaderAll.fillParticle(&pos); loaderAll.fillParticleAtMortonIndex(&pos, idxProc*NbParticles + idxPart,NbLevels); tree.insert(pos); }