Skip to content

S
ScalFMM
Commit e5fb1841 authored by Martin Khannouz's avatar Martin Khannouz Committed by Berenger Bramas
Browse files
Options

Got something working on almoste all cases. 

The implicit part reproducing explicit mapping works well.
But on the explicit side, it turn out that if all morton indexes ain't
used, it segfault.
Gotta investigate it.
parent 49284725
Hide whitespace changes
Inline Side-by-side
Showing with 28 additions and 53 deletions
Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp
View file @ e5fb1841
......@@ -287,8 +287,7 @@ public:
void syncData(){
for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){
for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){
//if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size()))) {
if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel)) {//Clean only our data handle
if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel)) {//Clean only our data handle
starpu_data_acquire(cellHandles[idxLevel][idxHandle].symb, STARPU_R);
starpu_data_release(cellHandles[idxLevel][idxHandle].symb);
starpu_data_acquire(cellHandles[idxLevel][idxHandle].up, STARPU_R);
......@@ -300,8 +299,7 @@ public:
}
{
for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){
//if(isDataOwned(idxHandle, int(particleHandles.size()))) {
if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1)) {//Clean only our data handle
if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1)) {//Clean only our data handle
starpu_data_acquire(particleHandles[idxHandle].symb, STARPU_R);
starpu_data_release(particleHandles[idxHandle].symb);
starpu_data_acquire(particleHandles[idxHandle].down, STARPU_R);
......@@ -435,13 +433,6 @@ public:
nodeRepartition[idxLevel][node_id][1] = nodeRepartition[idxLevel+1][node_id][1] >> 3;
}
}
//for(int idxLevel = 0; idxLevel < tree->getHeight(); ++idxLevel){
//cout << "Level " << idxLevel << endl;
//for(int node_id = 0; node_id < nproc; ++node_id){
//cout << "\t" << nodeRepartition[idxLevel][node_id][0] << " " << nodeRepartition[idxLevel][node_id][0] << endl;
//}
//}
}
protected:
/**
......@@ -842,8 +833,7 @@ protected:
void cleanHandle(){
for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){
for(int idxHandle = 0 ; idxHandle < int(cellHandles[idxLevel].size()) ; ++idxHandle){
//if(isDataOwned(idxHandle, int(cellHandles[idxLevel].size())))//Clean only our data handle
if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex()+1, idxLevel))//Clean only our data handle
if(isDataOwnedBerenger(tree->getCellGroup(idxLevel, idxHandle)->getStartingIndex(), idxLevel))//Clean only our data handle
{
starpu_data_unregister(cellHandles[idxLevel][idxHandle].symb);
starpu_data_unregister(cellHandles[idxLevel][idxHandle].up);
......@@ -854,8 +844,7 @@ protected:
}
{
for(int idxHandle = 0 ; idxHandle < int(particleHandles.size()) ; ++idxHandle){
//if(isDataOwned(idxHandle, int(particleHandles.size())))
if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex()+1, tree->getHeight()-1))//Clean only our data handle
if(isDataOwnedBerenger(tree->getCellGroup(tree->getHeight()-1, idxHandle)->getStartingIndex(), tree->getHeight()-1))//Clean only our data handle
{
starpu_data_unregister(particleHandles[idxHandle].symb);
starpu_data_unregister(particleHandles[idxHandle].down);
......@@ -875,10 +864,8 @@ protected:
cellHandles[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){
const CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup);
int registeringNode = dataMappingBerenger(currentCells->getStartingIndex()+1, idxLevel);
int registeringNode = dataMappingBerenger(currentCells->getStartingIndex(), idxLevel);
//int registeringNode = dataMapping(idxGroup, tree->getNbCellGroupAtLevel(idxLevel));
//cout << registeringNode << endl;
where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1;
starpu_variable_data_register(&cellHandles[idxLevel][idxGroup].symb, where,
(uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte());
......@@ -900,8 +887,7 @@ protected:
{
particleHandles.resize(tree->getNbParticleGroup());
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
//int registeringNode = dataMapping(idxGroup, tree->getNbParticleGroup());
int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex()+1, tree->getHeight()-1);
int registeringNode = dataMappingBerenger(tree->getCellGroup(tree->getHeight()-1, idxGroup)->getStartingIndex(), tree->getHeight()-1);
where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1;
ParticleGroupClass* containers = tree->getParticleGroup(idxGroup);
starpu_variable_data_register(&particleHandles[idxGroup].symb, where,
......@@ -924,12 +910,11 @@ protected:
}
}
int dataMappingBerenger(MortonIndex const idx, int const idxLevel) const {
//return idxLevel%4;
for(int i = 0; i < nproc; ++i)
if(nodeRepartition[idxLevel][i][0] <= nodeRepartition[idxLevel][i][1] && idx >= nodeRepartition[idxLevel][i][0] && idx <= nodeRepartition[idxLevel][i][1])
return i;
if(mpi_rank == 0)
cout << "Error !! [" << idx << "," << idxLevel << "]" << endl;
cout << "[scalfmm][map error] idx " << idx << " on level " << idxLevel << " isn't mapped on any proccess." << endl;
return -1;
}
/**
......
Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp
View file @ e5fb1841
......@@ -317,8 +317,14 @@ public:
~FGroupTaskStarPUMpiAlgorithm(){
starpu_resume();
std::cout << "Loutre " << comm.processId() << std::endl;
comm.barrier();
cleanHandle();
std::cout << "Canard " << comm.processId() << std::endl;
comm.barrier();
cleanHandleMpi();
std::cout << "Suricate " << comm.processId() << std::endl;
comm.barrier();
delete[] cellHandles;
starpu_pthread_mutex_t releaseMutex;
......@@ -1344,12 +1350,16 @@ protected:
for(int idxLevel = 0 ; idxLevel < int(remoteCellGroups.size()) ; ++idxLevel){
for(int idxHandle = 0 ; idxHandle < int(remoteCellGroups[idxLevel].size()) ; ++idxHandle){
if(remoteCellGroups[idxLevel][idxHandle].ptrSymb){
if(comm.processId() == 4)
std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl;
starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleSymb);
starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleUp);
FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrSymb);
FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrUp);
if(remoteCellGroups[idxLevel][idxHandle].ptrDown){
if(comm.processId() == 4)
std::cout << "Narval " << idxLevel << " " << idxHandle << std::endl;
starpu_data_unregister(remoteCellGroups[idxLevel][idxHandle].handleDown);
FAlignedMemory::DeallocBytes(remoteCellGroups[idxLevel][idxHandle].ptrDown);
}
......@@ -1360,12 +1370,16 @@ protected:
{
for(int idxHandle = 0 ; idxHandle < int(remoteParticleGroupss.size()) ; ++idxHandle){
if(remoteParticleGroupss[idxHandle].ptrSymb){
if(comm.processId() == 4)
std::cout << "Narval part " << idxHandle << std::endl;
starpu_data_unregister(remoteParticleGroupss[idxHandle].handleSymb);
FAlignedMemory::DeallocBytes(remoteParticleGroupss[idxHandle].ptrSymb);
}
}
remoteParticleGroupss.clear();
}
if(comm.processId() == 4)
std::cout << "Done" << std::endl;
}
////////////////////////////////////////////////////////////////////////////
......
Tests/GroupTree/testBlockedImplicitAlgorithm.cpp
View file @ e5fb1841
......@@ -65,7 +65,7 @@ using namespace std;
// FFmmAlgorithmTask FFmmAlgorithmThread
typedef FFmmAlgorithm<OctreeClass, CellClass, ContainerClass, KernelClass, LeafClass > FmmClass;
//#define LOAD_FILE
#define LOAD_FILE
#ifndef LOAD_FILE
typedef FRandomLoader<FReal> LoaderClass;
#else
......@@ -100,8 +100,8 @@ int main(int argc, char* argv[]){
cout << "Pas de mpi -_-\" " << endl;
#endif
#ifndef LOAD_FILE
const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(100));
LoaderClass loader(NbParticles, 1.0, FPoint<FReal>(0,0,0), 0);
const FSize NbParticles = FParameters::getValue(argc,argv,FParameterDefinitions::NbParticles.options, FSize(10000));
LoaderClass loader(NbParticles, 1.0, FPoint<FReal>(0,0,0), 0);
#else
// Load the particles
const char* const filename = FParameters::getStr(argc,argv,FParameterDefinitions::InputFile.options, "../Data/test20k.fma");
......@@ -157,15 +157,6 @@ int main(int argc, char* argv[]){
mpi_rank = groupalgo.getRank();
cout << "Executing time (implicit node " << mpi_rank << ") " << elapsedTime << "s\n";
timeAverage(mpi_rank, nproc, elapsedTime);
//if( groupalgo.getRank() != 0)
//return 0;
//groupedTree.printInfoBlocks();
//for(int i = 0; i < NbLevels; ++i)
//{
//cout << "Level " << i << "(" << sizeForEachGroup[i].size() << ")" << endl;
//for(int j = 0; j < sizeForEachGroup[i].size(); ++j)
//cout << "\t" << sizeForEachGroup[i][j] << endl;
//}
// Usual algorithm
KernelClass kernels; // FTestKernels FBasicKernels
......@@ -276,7 +267,7 @@ void sortParticle(FPoint<FReal> * allParticles, int treeHeight, int groupSize, v
}
//Compte le nombre de feuilles
sizeForEachGroup.resize(treeHeight+1);//Le +1 est pour les particules
sizeForEachGroup.resize(treeHeight);
MortonIndex previousLeaf = -1;
int numberOfLeaf = 0;
for(FSize idxPart = 0 ; idxPart < nbParticles ; ++idxPart)
......@@ -331,10 +322,6 @@ void sortParticle(FPoint<FReal> * allParticles, int treeHeight, int groupSize, v
std::vector<std::vector<std::vector<MortonIndex>>> nodeRepartition;
createNodeRepartition(distributedMortonIndex, nodeRepartition, nproc, treeHeight);
//for(int i = 0; i < nproc; ++i)
//{
//cout << nodeRepartition[1][i][0] << " - " << nodeRepartition[1][i][1] << endl;
//}
//Pour chaque niveau calcul de la taille des groupe
for(int idxLevel = treeHeight - 2; idxLevel >= 0; --idxLevel)
{
......
Tests/GroupTree/testBlockedMpiAlgorithm.cpp
View file @ e5fb1841
......@@ -104,7 +104,7 @@ int main(int argc, char* argv[]){
memset(particles.get(), 0, sizeof(TestParticle) * loader.getNumberOfParticles());
for(FSize idxPart = 0 ; idxPart < loader.getNumberOfParticles() ; ++idxPart){
loader.fillParticleAtMortonIndex(&(particles[idxPart].position), mpiComm.global().processId()*NbParticles + idxPart,NbLevels);
//loader.fillParticle(&(particles[idxPart].position));
//loader.fillParticle(&(particles[idxPart].position));
}
// Sort in parallel
FVector<TestParticle> myParticles;
......@@ -147,28 +147,20 @@ int main(int argc, char* argv[]){
std::cout << "Exchange particle to create the file" << std::endl;
std::vector<TestParticle*> particlesGathered;
std::vector<int> sizeGathered;
std::vector<MortonIndex> mortonIndex;
//Ajout des mes particules
int sizeofParticle = sizeof(TestParticle)*myParticles.getSize();
sizeGathered.push_back(sizeofParticle);
particlesGathered.push_back(new TestParticle[sizeofParticle]);
memcpy(particlesGathered.back(), myParticles.data(), sizeofParticle);
mortonIndex.push_back(leftLimite);
mortonIndex.push_back(myLeftLimite);
//Recupération des particules des autres
for(int i = 1; i < mpiComm.global().processCount(); ++i)
{
int sizeReceive;
MortonIndex mortonStart, mortonEnd;
MPI_Recv(&sizeReceive, sizeof(sizeReceive), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE);
sizeGathered.push_back(sizeReceive);
particlesGathered.push_back(new TestParticle[sizeReceive]);
MPI_Recv(particlesGathered.back(), sizeReceive, MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE);
MPI_Recv(&mortonStart, sizeof(mortonStart), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE);
MPI_Recv(&mortonEnd, sizeof(mortonEnd), MPI_BYTE, i, 0, mpiComm.global().getComm(), MPI_STATUS_IGNORE);
mortonIndex.push_back(mortonStart);
mortonIndex.push_back(mortonEnd);
}
int sum = 0;
for(int a : sizeGathered)
......@@ -180,10 +172,6 @@ int main(int argc, char* argv[]){
writer.writeHeader(loader.getCenterOfBox(), loader.getBoxWidth(),totalNbParticles, particles[0]);
for(unsigned int i = 0; i < particlesGathered.size(); ++i)
writer.writeArrayOfParticles(particlesGathered[i], sizeGathered[i]/sizeof(TestParticle));
std::ofstream fichier("mapping", std::ios::out | std::ios::trunc); //déclaration du flux et ouverture du fichier
fichier << mortonIndex.size()/2 << std::endl;
for(unsigned int i = 0; i < mortonIndex.size(); i+=2)
fichier << mortonIndex[i] << " " << mortonIndex[i+1] << std::endl;
for(TestParticle* ptr : particlesGathered)
delete ptr;
std::cout << "Done exchanging !" << std::endl;
......@@ -221,6 +209,7 @@ int main(int argc, char* argv[]){
});
mpiComm.global().barrier();
typedef FTestCell CellClass;
typedef FTestParticleContainer<FReal> ContainerClass;
......@@ -236,7 +225,7 @@ int main(int argc, char* argv[]){
FRandomLoader<FReal> loaderAll(NbParticles, 1.0, FPoint<FReal>(0,0,0), idxProc);
for(FSize idxPart = 0 ; idxPart < loaderAll.getNumberOfParticles() ; ++idxPart){
FPoint<FReal> pos;
//loaderAll.fillParticle(&pos);
//loaderAll.fillParticle(&pos);
loaderAll.fillParticleAtMortonIndex(&pos, idxProc*NbParticles + idxPart,NbLevels);
tree.insert(pos);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment