Skip to content

S
ScalFMM
Commit 912087bb authored by Martin Khannouz's avatar Martin Khannouz Committed by Berenger Bramas
Browse files
Options

MPI distribution of morton indexes are compute within 

testBlockedImplicitAlgorithm and doesn't require a mapping file anymore.

To do it properly, MPI_Init is done in the main and MPI_Finalize in the
algorithm destructor.
parent 30ed1dc6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 129 additions and 147 deletions
Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp
View file @ 912087bb
......@@ -186,7 +186,7 @@ public:
PrioClass::Controller().init(&conf, tree->getHeight(), inKernels);
#endif
FAssertLF(starpu_init(&conf) == 0);
FAssertLF(starpu_mpi_init ( 0, 0, 1 ) == 0);
FAssertLF(starpu_mpi_init ( 0, 0, 0 ) == 0);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD,&nproc);
#ifdef STARPU_USE_TASK_NAME
......@@ -350,8 +350,9 @@ public:
starpu_arbiter_destroy(arbiterGlobal);
#endif
starpu_mpi_shutdown();
starpu_mpi_shutdown();
starpu_shutdown();
MPI_Finalize(); //TODO put it somewhere else, it's really dirty here.
}
void rebuildInteractions(){
......@@ -434,6 +435,13 @@ public:
nodeRepartition[idxLevel][node_id][1] = nodeRepartition[idxLevel+1][node_id][1] >> 3;
}
}
//for(int idxLevel = 0; idxLevel < tree->getHeight(); ++idxLevel){
//cout << "Level " << idxLevel << endl;
//for(int node_id = 0; node_id < nproc; ++node_id){
//cout << "\t" << nodeRepartition[idxLevel][node_id][0] << " " << nodeRepartition[idxLevel][node_id][0] << endl;
//}
//}
}
protected:
/**
......@@ -1586,8 +1594,8 @@ protected:
for(int idxInteraction = 0; idxInteraction < int(externalInteractionsLeafLevel[idxGroup].size()) ; ++idxInteraction){
const int interactionid = externalInteractionsLeafLevel[idxGroup][idxInteraction].otherBlockId;
const std::vector<OutOfBlockInteraction>* outsideInteractions = &externalInteractionsLeafLevel[idxGroup][idxInteraction].interactions;
if(starpu_mpi_data_get_rank(particleHandles[idxGroup].down) == starpu_mpi_data_get_rank(particleHandles[interactionid].down))
{
//if(starpu_mpi_data_get_rank(particleHandles[idxGroup].down) == starpu_mpi_data_get_rank(particleHandles[interactionid].down))
//{
starpu_mpi_insert_task(MPI_COMM_WORLD,
&p2p_cl_inout,
STARPU_VALUE, &wrapperptr, sizeof(wrapperptr),
......@@ -1630,74 +1638,74 @@ protected:
#endif
#endif
0);
}
else
{
starpu_mpi_insert_task(MPI_COMM_WORLD,
&p2p_cl_inout_mpi,
STARPU_VALUE, &wrapperptr, sizeof(wrapperptr),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_VALUE, &particleHandles[idxGroup].intervalSize, sizeof(int),
#ifdef SCALFMM_STARPU_USE_PRIO
STARPU_PRIORITY, PrioClass::Controller().getInsertionPosP2PExtern(),
#endif
STARPU_R, particleHandles[idxGroup].symb,
(STARPU_RW|STARPU_COMMUTE_IF_SUPPORTED), particleHandles[idxGroup].down,
STARPU_R, particleHandles[interactionid].symb,
#ifdef STARPU_USE_TASK_NAME
#ifndef SCALFMM_SIMGRID_TASKNAMEPARAMS
STARPU_NAME, p2pOuterTaskNames.get(),
#else
//"P2P_out-nb_i_p_nb_i_p_s"
STARPU_NAME, taskNames->print("P2P_out", "%d, %lld, %lld, %d, %lld, %lld, %d, %lld, %lld, %lld, %lld, %d\n",
tree->getParticleGroup(idxGroup)->getNumberOfLeavesInBlock(),
tree->getParticleGroup(idxGroup)->getSizeOfInterval(),
tree->getParticleGroup(idxGroup)->getNbParticlesInGroup(),
tree->getParticleGroup(interactionid)->getNumberOfLeavesInBlock(),
tree->getParticleGroup(interactionid)->getSizeOfInterval(),
tree->getParticleGroup(interactionid)->getNbParticlesInGroup(),
outsideInteractions->size(),
tree->getParticleGroup(idxGroup)->getStartingIndex(),
tree->getParticleGroup(idxGroup)->getEndingIndex(),
tree->getParticleGroup(interactionid)->getStartingIndex(),
tree->getParticleGroup(interactionid)->getEndingIndex(),
starpu_mpi_data_get_rank(particleHandles[idxGroup].down)),
#endif
#endif
0);
starpu_mpi_insert_task(MPI_COMM_WORLD,
&p2p_cl_inout_mpi,
STARPU_VALUE, &wrapperptr, sizeof(wrapperptr),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_VALUE, &particleHandles[idxGroup].intervalSize, sizeof(int),
#ifdef SCALFMM_STARPU_USE_PRIO
STARPU_PRIORITY, PrioClass::Controller().getInsertionPosP2PExtern(),
#endif
STARPU_R, particleHandles[interactionid].symb,
(STARPU_RW|STARPU_COMMUTE_IF_SUPPORTED), particleHandles[interactionid].down,
STARPU_R, particleHandles[idxGroup].symb,
#ifdef STARPU_USE_TASK_NAME
#ifndef SCALFMM_SIMGRID_TASKNAMEPARAMS
STARPU_NAME, p2pOuterTaskNames.get(),
#else
//"P2P_out-nb_i_p_nb_i_p_s"
STARPU_NAME, taskNames->print("P2P_out", "%d, %lld, %lld, %d, %lld, %lld, %d, %lld, %lld, %lld, %lld, %d\n",
tree->getParticleGroup(interactionid)->getNumberOfLeavesInBlock(),
tree->getParticleGroup(interactionid)->getSizeOfInterval(),
tree->getParticleGroup(interactionid)->getNbParticlesInGroup(),
tree->getParticleGroup(idxGroup)->getNumberOfLeavesInBlock(),
tree->getParticleGroup(idxGroup)->getSizeOfInterval(),
tree->getParticleGroup(idxGroup)->getNbParticlesInGroup(),
outsideInteractions->size(),
tree->getParticleGroup(interactionid)->getStartingIndex(),
tree->getParticleGroup(interactionid)->getEndingIndex(),
tree->getParticleGroup(idxGroup)->getStartingIndex(),
tree->getParticleGroup(idxGroup)->getEndingIndex(),
starpu_mpi_data_get_rank(particleHandles[interactionid].down)),
#endif
#endif
0);
}
//}
//else
//{
//starpu_mpi_insert_task(MPI_COMM_WORLD,
//&p2p_cl_inout_mpi,
//STARPU_VALUE, &wrapperptr, sizeof(wrapperptr),
//STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
//STARPU_VALUE, &particleHandles[idxGroup].intervalSize, sizeof(int),
//#ifdef SCALFMM_STARPU_USE_PRIO
//STARPU_PRIORITY, PrioClass::Controller().getInsertionPosP2PExtern(),
//#endif
//STARPU_R, particleHandles[idxGroup].symb,
//(STARPU_RW|STARPU_COMMUTE_IF_SUPPORTED), particleHandles[idxGroup].down,
//STARPU_R, particleHandles[interactionid].symb,
//#ifdef STARPU_USE_TASK_NAME
//#ifndef SCALFMM_SIMGRID_TASKNAMEPARAMS
//STARPU_NAME, p2pOuterTaskNames.get(),
//#else
////"P2P_out-nb_i_p_nb_i_p_s"
//STARPU_NAME, taskNames->print("P2P_out", "%d, %lld, %lld, %d, %lld, %lld, %d, %lld, %lld, %lld, %lld, %d\n",
//tree->getParticleGroup(idxGroup)->getNumberOfLeavesInBlock(),
//tree->getParticleGroup(idxGroup)->getSizeOfInterval(),
//tree->getParticleGroup(idxGroup)->getNbParticlesInGroup(),
//tree->getParticleGroup(interactionid)->getNumberOfLeavesInBlock(),
//tree->getParticleGroup(interactionid)->getSizeOfInterval(),
//tree->getParticleGroup(interactionid)->getNbParticlesInGroup(),
//outsideInteractions->size(),
//tree->getParticleGroup(idxGroup)->getStartingIndex(),
//tree->getParticleGroup(idxGroup)->getEndingIndex(),
//tree->getParticleGroup(interactionid)->getStartingIndex(),
//tree->getParticleGroup(interactionid)->getEndingIndex(),
//starpu_mpi_data_get_rank(particleHandles[idxGroup].down)),
//#endif
//#endif
//0);
//starpu_mpi_insert_task(MPI_COMM_WORLD,
//&p2p_cl_inout_mpi,
//STARPU_VALUE, &wrapperptr, sizeof(wrapperptr),
//STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
//STARPU_VALUE, &particleHandles[idxGroup].intervalSize, sizeof(int),
//#ifdef SCALFMM_STARPU_USE_PRIO
//STARPU_PRIORITY, PrioClass::Controller().getInsertionPosP2PExtern(),
//#endif
//STARPU_R, particleHandles[interactionid].symb,
//(STARPU_RW|STARPU_COMMUTE_IF_SUPPORTED), particleHandles[interactionid].down,
//STARPU_R, particleHandles[idxGroup].symb,
//#ifdef STARPU_USE_TASK_NAME
//#ifndef SCALFMM_SIMGRID_TASKNAMEPARAMS
//STARPU_NAME, p2pOuterTaskNames.get(),
//#else
////"P2P_out-nb_i_p_nb_i_p_s"
//STARPU_NAME, taskNames->print("P2P_out", "%d, %lld, %lld, %d, %lld, %lld, %d, %lld, %lld, %lld, %lld, %d\n",
//tree->getParticleGroup(interactionid)->getNumberOfLeavesInBlock(),
//tree->getParticleGroup(interactionid)->getSizeOfInterval(),
//tree->getParticleGroup(interactionid)->getNbParticlesInGroup(),
//tree->getParticleGroup(idxGroup)->getNumberOfLeavesInBlock(),
//tree->getParticleGroup(idxGroup)->getSizeOfInterval(),
//tree->getParticleGroup(idxGroup)->getNbParticlesInGroup(),
//outsideInteractions->size(),
//tree->getParticleGroup(interactionid)->getStartingIndex(),
//tree->getParticleGroup(interactionid)->getEndingIndex(),
//tree->getParticleGroup(idxGroup)->getStartingIndex(),
//tree->getParticleGroup(idxGroup)->getEndingIndex(),
//starpu_mpi_data_get_rank(particleHandles[interactionid].down)),
//#endif
//#endif
//0);
//}
}
}
FLOG( timerOutBlock.tac() );
......@@ -1806,9 +1814,6 @@ protected:
}
}
#endif
};
#endif // FGROUPTASKSTARPUALGORITHM_HPP
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment