Skip to content

GitLab

Une MAJ de sécurité est nécessaire sur notre version actuelle. Elle sera effectuée lundi 02/08 entre 12h30 et 13h. L'interruption de service devrait durer quelques minutes (probablement moins de 5 minutes).

Commit 7ddc5545 authored by BRAMAS Berenger's avatar BRAMAS Berenger
Browse files

use two kind of handles for the shared memory starpu

parent 6bc5a733
Hide whitespace changes
Inline Side-by-side
Showing with 61 additions and 32 deletions
Src/GroupTree/FGroupTaskStarpuAlgorithm.hpp
View file @ 7ddc5545
......@@ -10,6 +10,7 @@
#include "../Containers/FTreeCoordinate.hpp"
#include "../Utils/FLog.hpp"
#include "../Utils/FTic.hpp"
#include "../Utils/FAssert.hpp"
#include <vector>
#include <vector>
......@@ -50,7 +51,8 @@ protected:
KernelClass** kernels; //< The kernels
ThisClass* thisptr;
std::vector<starpu_data_handle_t>* handles;
std::vector<starpu_data_handle_t>* handles_up;
std::vector<starpu_data_handle_t>* handles_down;
starpu_codelet p2m_cl;
starpu_codelet m2m_cl[9];
......@@ -66,7 +68,7 @@ protected:
public:
FGroupTaskStarPUAlgorithm(OctreeClass*const inTree, KernelClass* inKernels, const int inMaxThreads = -1)
: MaxThreads(inMaxThreads), tree(inTree), kernels(nullptr),
thisptr(this), handles(nullptr){
thisptr(this), handles_up(nullptr), handles_down(nullptr){
FAssertLF(tree, "tree cannot be null");
FAssertLF(inKernels, "kernels cannot be null");
FAssertLF(MaxThreads <= STARPU_MAXCPUS, "number of threads to high");
......@@ -79,7 +81,9 @@ public:
MaxThreads = starpu_worker_get_count();//starpu_cpu_worker_get_count();
handles = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
handles_up = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
handles_down = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
kernels = new KernelClass*[MaxThreads];
for(int idxThread = 0 ; idxThread < MaxThreads ; ++idxThread){
this->kernels[idxThread] = new KernelClass(*inKernels);
......@@ -97,7 +101,8 @@ public:
delete[] kernels;
cleanHandle();
delete[] handles;
delete[] handles_up;
delete[] handles_down;
starpu_resume();
starpu_shutdown();
......@@ -182,30 +187,41 @@ protected:
memset(&m2l_cl_in, 0, sizeof(m2l_cl_in));
m2l_cl_in.where = STARPU_CPU;
m2l_cl_in.cpu_funcs[0] = transferInPassCallback;
m2l_cl_in.nbuffers = 1;
m2l_cl_in.nbuffers = 2;
m2l_cl_in.modes[0] = STARPU_RW;
m2l_cl_in.modes[1] = STARPU_R;
memset(&m2l_cl_inout, 0, sizeof(m2l_cl_inout));
m2l_cl_inout.where = STARPU_CPU;
m2l_cl_inout.cpu_funcs[0] = transferInoutPassCallback;
m2l_cl_inout.nbuffers = 2;
m2l_cl_inout.nbuffers = 4;
m2l_cl_inout.modes[0] = STARPU_RW;
m2l_cl_inout.modes[1] = STARPU_RW;
m2l_cl_inout.modes[2] = STARPU_R;
m2l_cl_inout.modes[3] = STARPU_R;
}
/** dealloc in a starpu way all the defined handles */
void cleanHandle(){
for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){
for(int idxHandle = 0 ; idxHandle < int(handles[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles[idxLevel][idxHandle]);
for(int idxHandle = 0 ; idxHandle < int(handles_up[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_up[idxLevel][idxHandle]);
}
handles_up[idxLevel].clear();
for(int idxHandle = 0 ; idxHandle < int(handles_down[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_down[idxLevel][idxHandle]);
}
handles[idxLevel].clear();
handles_down[idxLevel].clear();
}
{
const int idxLevel = tree->getHeight();
for(int idxHandle = 0 ; idxHandle < int(handles[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles[idxLevel][idxHandle]);
for(int idxHandle = 0 ; idxHandle < int(handles_up[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_up[idxLevel][idxHandle]);
}
handles_up[idxLevel].clear();
for(int idxHandle = 0 ; idxHandle < int(handles_down[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_down[idxLevel][idxHandle]);
}
handles[idxLevel].clear();
handles_down[idxLevel].clear();
}
}
......@@ -216,20 +232,26 @@ protected:
cleanHandle();
for(int idxLevel = 2 ; idxLevel < tree->getHeight() ; ++idxLevel){
handles[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
handles_up[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
handles_down[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){
const CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup);
starpu_variable_data_register(&handles[idxLevel][idxGroup], 0,
starpu_variable_data_register(&handles_up[idxLevel][idxGroup], 0,
(uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte());
starpu_variable_data_register(&handles_down[idxLevel][idxGroup], 0,
(uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte());
}
}
{
const int idxLevel = tree->getHeight();
handles[idxLevel].resize(tree->getNbParticleGroup());
handles_up[idxLevel].resize(tree->getNbParticleGroup());
handles_down[idxLevel].resize(tree->getNbParticleGroup());
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
ParticleGroupClass* containers = tree->getParticleGroup(idxGroup);
starpu_variable_data_register(&handles[idxLevel][idxGroup], 0,
starpu_variable_data_register(&handles_up[idxLevel][idxGroup], 0,
(uintptr_t)containers->getRawBuffer(), containers->getBufferSizeInByte());
starpu_variable_data_register(&handles_down[idxLevel][idxGroup], 0,
(uintptr_t)containers->getRawBuffer(), containers->getBufferSizeInByte());
}
}
......@@ -421,8 +443,8 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&p2m_cl,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_RW, handles[tree->getHeight()-1][idxGroup],
STARPU_R, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles_up[tree->getHeight()-1][idxGroup],
STARPU_R, handles_up[tree->getHeight()][idxGroup],
0);
}
......@@ -470,7 +492,7 @@ protected:
struct starpu_task* const task = starpu_task_create();
task->dyn_handles = (starpu_data_handle_t*)malloc(sizeof(starpu_data_handle_t)*10);
task->dyn_handles[0] = handles[idxLevel][idxGroup];
task->dyn_handles[0] = handles_up[idxLevel][idxGroup];
// Skip current group if needed
if( tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= (currentCells->getStartingIndex()<<3) ){
......@@ -480,13 +502,13 @@ protected:
}
// Copy at max 8 groups
int nbSubCellGroups = 0;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_up[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
while(tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= ((currentCells->getEndingIndex()<<3)+7)
&& (idxSubGroup+1) != tree->getNbCellGroupAtLevel(idxLevel+1)
&& tree->getCellGroup(idxLevel+1, idxSubGroup+1)->getStartingIndex() <= (currentCells->getEndingIndex()<<3)+7 ){
idxSubGroup += 1;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_up[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
FAssertLF( nbSubCellGroups <= 9 );
}
......@@ -575,7 +597,8 @@ protected:
starpu_insert_task(&m2l_cl_in,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &idxLevel, sizeof(idxLevel),
STARPU_RW, handles[idxLevel][idxGroup],
STARPU_RW, handles_down[idxLevel][idxGroup],
STARPU_R, handles_up[idxLevel][idxGroup],
0);
}
FLOG( timerInBlock.tac() );
......@@ -590,8 +613,10 @@ protected:
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &idxLevel, sizeof(idxLevel),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_RW, handles[idxLevel][idxGroup],
STARPU_RW, handles[idxLevel][interactionid],
STARPU_RW, handles_down[idxLevel][idxGroup],
STARPU_RW, handles_down[idxLevel][interactionid],
STARPU_R, handles_up[idxLevel][idxGroup],
STARPU_R, handles_up[idxLevel][interactionid],
0);
}
}
......@@ -603,6 +628,7 @@ protected:
}
static void transferInPassCallback(void *buffers[], void *cl_arg){
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[0]) == STARPU_VARIABLE_GET_PTR(buffers[1]));
CellContainerClass currentCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[0]),
STARPU_VARIABLE_GET_ELEMSIZE(buffers[0]));
......@@ -649,6 +675,9 @@ protected:
}
static void transferInoutPassCallback(void *buffers[], void *cl_arg){
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[0]) == STARPU_VARIABLE_GET_PTR(buffers[2]));
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[1]) == STARPU_VARIABLE_GET_PTR(buffers[3]));
CellContainerClass currentCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[0]),
STARPU_VARIABLE_GET_ELEMSIZE(buffers[0]));
CellContainerClass externalCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[1]),
......@@ -704,7 +733,7 @@ protected:
struct starpu_task* const task = starpu_task_create();
task->dyn_handles = (starpu_data_handle_t*)malloc(sizeof(starpu_data_handle_t)*10);
task->dyn_handles[0] = handles[idxLevel][idxGroup];
task->dyn_handles[0] = handles_down[idxLevel][idxGroup];
// Skip current group if needed
if( tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= (currentCells->getStartingIndex()<<3) ){
......@@ -714,13 +743,13 @@ protected:
}
// Copy at max 8 groups
int nbSubCellGroups = 0;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_down[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
while(tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= ((currentCells->getEndingIndex()<<3)+7)
&& (idxSubGroup+1) != tree->getNbCellGroupAtLevel(idxLevel+1)
&& tree->getCellGroup(idxLevel+1, idxSubGroup+1)->getStartingIndex() <= (currentCells->getEndingIndex()<<3)+7 ){
idxSubGroup += 1;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_down[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
FAssertLF( nbSubCellGroups <= 9 );
}
......@@ -808,7 +837,7 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&p2p_cl_in,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
0);
}
FLOG( timerInBlock.tac() );
......@@ -820,8 +849,8 @@ protected:
starpu_insert_task(&p2p_cl_inout,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles[tree->getHeight()][interactionid],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][interactionid],
0);
}
}
......@@ -920,8 +949,8 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&l2p_cl,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_R, handles[tree->getHeight()-1][idxGroup],
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_R, handles_down[tree->getHeight()-1][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
0);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment