Skip to content
GitLab
Commit 7ddc5545 authored by BRAMAS Berenger's avatar BRAMAS Berenger
Browse files

use two kind of handles for the shared memory starpu

parent 6bc5a733
Hide whitespace changes
Inline Side-by-side
Src/GroupTree/FGroupTaskStarpuAlgorithm.hpp
View file @ 7ddc5545
......@@ -10,6 +10,7 @@
#include "../Containers/FTreeCoordinate.hpp"
#include "../Utils/FLog.hpp"
#include "../Utils/FTic.hpp"
#include "../Utils/FAssert.hpp"
#include <vector>
#include <vector>
......@@ -50,7 +51,8 @@ protected:
KernelClass** kernels; //< The kernels
ThisClass* thisptr;
std::vector<starpu_data_handle_t>* handles;
std::vector<starpu_data_handle_t>* handles_up;
std::vector<starpu_data_handle_t>* handles_down;
starpu_codelet p2m_cl;
starpu_codelet m2m_cl[9];
......@@ -66,7 +68,7 @@ protected:
public:
FGroupTaskStarPUAlgorithm(OctreeClass*const inTree, KernelClass* inKernels, const int inMaxThreads = -1)
: MaxThreads(inMaxThreads), tree(inTree), kernels(nullptr),
thisptr(this), handles(nullptr){
thisptr(this), handles_up(nullptr), handles_down(nullptr){
FAssertLF(tree, "tree cannot be null");
FAssertLF(inKernels, "kernels cannot be null");
FAssertLF(MaxThreads <= STARPU_MAXCPUS, "number of threads to high");
......@@ -79,7 +81,9 @@ public:
MaxThreads = starpu_worker_get_count();//starpu_cpu_worker_get_count();
handles = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
handles_up = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
handles_down = new std::vector<starpu_data_handle_t>[tree->getHeight()+1];
kernels = new KernelClass*[MaxThreads];
for(int idxThread = 0 ; idxThread < MaxThreads ; ++idxThread){
this->kernels[idxThread] = new KernelClass(*inKernels);
......@@ -97,7 +101,8 @@ public:
delete[] kernels;
cleanHandle();
delete[] handles;
delete[] handles_up;
delete[] handles_down;
starpu_resume();
starpu_shutdown();
......@@ -182,30 +187,41 @@ protected:
memset(&m2l_cl_in, 0, sizeof(m2l_cl_in));
m2l_cl_in.where = STARPU_CPU;
m2l_cl_in.cpu_funcs[0] = transferInPassCallback;
m2l_cl_in.nbuffers = 1;
m2l_cl_in.nbuffers = 2;
m2l_cl_in.modes[0] = STARPU_RW;
m2l_cl_in.modes[1] = STARPU_R;
memset(&m2l_cl_inout, 0, sizeof(m2l_cl_inout));
m2l_cl_inout.where = STARPU_CPU;
m2l_cl_inout.cpu_funcs[0] = transferInoutPassCallback;
m2l_cl_inout.nbuffers = 2;
m2l_cl_inout.nbuffers = 4;
m2l_cl_inout.modes[0] = STARPU_RW;
m2l_cl_inout.modes[1] = STARPU_RW;
m2l_cl_inout.modes[2] = STARPU_R;
m2l_cl_inout.modes[3] = STARPU_R;
}
/** dealloc in a starpu way all the defined handles */
void cleanHandle(){
for(int idxLevel = 0 ; idxLevel < tree->getHeight() ; ++idxLevel){
for(int idxHandle = 0 ; idxHandle < int(handles[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles[idxLevel][idxHandle]);
for(int idxHandle = 0 ; idxHandle < int(handles_up[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_up[idxLevel][idxHandle]);
}
handles_up[idxLevel].clear();
for(int idxHandle = 0 ; idxHandle < int(handles_down[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_down[idxLevel][idxHandle]);
}
handles[idxLevel].clear();
handles_down[idxLevel].clear();
}
{
const int idxLevel = tree->getHeight();
for(int idxHandle = 0 ; idxHandle < int(handles[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles[idxLevel][idxHandle]);
for(int idxHandle = 0 ; idxHandle < int(handles_up[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_up[idxLevel][idxHandle]);
}
handles_up[idxLevel].clear();
for(int idxHandle = 0 ; idxHandle < int(handles_down[idxLevel].size()) ; ++idxHandle){
starpu_data_unregister(handles_down[idxLevel][idxHandle]);
}
handles[idxLevel].clear();
handles_down[idxLevel].clear();
}
}
......@@ -216,20 +232,26 @@ protected:
cleanHandle();
for(int idxLevel = 2 ; idxLevel < tree->getHeight() ; ++idxLevel){
handles[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
handles_up[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
handles_down[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){
const CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup);
starpu_variable_data_register(&handles[idxLevel][idxGroup], 0,
starpu_variable_data_register(&handles_up[idxLevel][idxGroup], 0,
(uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte());
starpu_variable_data_register(&handles_down[idxLevel][idxGroup], 0,
(uintptr_t)currentCells->getRawBuffer(), currentCells->getBufferSizeInByte());
}
}
{
const int idxLevel = tree->getHeight();
handles[idxLevel].resize(tree->getNbParticleGroup());
handles_up[idxLevel].resize(tree->getNbParticleGroup());
handles_down[idxLevel].resize(tree->getNbParticleGroup());
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
ParticleGroupClass* containers = tree->getParticleGroup(idxGroup);
starpu_variable_data_register(&handles[idxLevel][idxGroup], 0,
starpu_variable_data_register(&handles_up[idxLevel][idxGroup], 0,
(uintptr_t)containers->getRawBuffer(), containers->getBufferSizeInByte());
starpu_variable_data_register(&handles_down[idxLevel][idxGroup], 0,
(uintptr_t)containers->getRawBuffer(), containers->getBufferSizeInByte());
}
}
......@@ -421,8 +443,8 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&p2m_cl,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_RW, handles[tree->getHeight()-1][idxGroup],
STARPU_R, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles_up[tree->getHeight()-1][idxGroup],
STARPU_R, handles_up[tree->getHeight()][idxGroup],
0);
}
......@@ -470,7 +492,7 @@ protected:
struct starpu_task* const task = starpu_task_create();
task->dyn_handles = (starpu_data_handle_t*)malloc(sizeof(starpu_data_handle_t)*10);
task->dyn_handles[0] = handles[idxLevel][idxGroup];
task->dyn_handles[0] = handles_up[idxLevel][idxGroup];
// Skip current group if needed
if( tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= (currentCells->getStartingIndex()<<3) ){
......@@ -480,13 +502,13 @@ protected:
}
// Copy at max 8 groups
int nbSubCellGroups = 0;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_up[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
while(tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= ((currentCells->getEndingIndex()<<3)+7)
&& (idxSubGroup+1) != tree->getNbCellGroupAtLevel(idxLevel+1)
&& tree->getCellGroup(idxLevel+1, idxSubGroup+1)->getStartingIndex() <= (currentCells->getEndingIndex()<<3)+7 ){
idxSubGroup += 1;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_up[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
FAssertLF( nbSubCellGroups <= 9 );
}
......@@ -575,7 +597,8 @@ protected:
starpu_insert_task(&m2l_cl_in,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &idxLevel, sizeof(idxLevel),
STARPU_RW, handles[idxLevel][idxGroup],
STARPU_RW, handles_down[idxLevel][idxGroup],
STARPU_R, handles_up[idxLevel][idxGroup],
0);
}
FLOG( timerInBlock.tac() );
......@@ -590,8 +613,10 @@ protected:
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &idxLevel, sizeof(idxLevel),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_RW, handles[idxLevel][idxGroup],
STARPU_RW, handles[idxLevel][interactionid],
STARPU_RW, handles_down[idxLevel][idxGroup],
STARPU_RW, handles_down[idxLevel][interactionid],
STARPU_R, handles_up[idxLevel][idxGroup],
STARPU_R, handles_up[idxLevel][interactionid],
0);
}
}
......@@ -603,6 +628,7 @@ protected:
}
static void transferInPassCallback(void *buffers[], void *cl_arg){
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[0]) == STARPU_VARIABLE_GET_PTR(buffers[1]));
CellContainerClass currentCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[0]),
STARPU_VARIABLE_GET_ELEMSIZE(buffers[0]));
......@@ -649,6 +675,9 @@ protected:
}
static void transferInoutPassCallback(void *buffers[], void *cl_arg){
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[0]) == STARPU_VARIABLE_GET_PTR(buffers[2]));
FAssertLF(STARPU_VARIABLE_GET_PTR(buffers[1]) == STARPU_VARIABLE_GET_PTR(buffers[3]));
CellContainerClass currentCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[0]),
STARPU_VARIABLE_GET_ELEMSIZE(buffers[0]));
CellContainerClass externalCells((unsigned char*)STARPU_VARIABLE_GET_PTR(buffers[1]),
......@@ -704,7 +733,7 @@ protected:
struct starpu_task* const task = starpu_task_create();
task->dyn_handles = (starpu_data_handle_t*)malloc(sizeof(starpu_data_handle_t)*10);
task->dyn_handles[0] = handles[idxLevel][idxGroup];
task->dyn_handles[0] = handles_down[idxLevel][idxGroup];
// Skip current group if needed
if( tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= (currentCells->getStartingIndex()<<3) ){
......@@ -714,13 +743,13 @@ protected:
}
// Copy at max 8 groups
int nbSubCellGroups = 0;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_down[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
while(tree->getCellGroup(idxLevel+1, idxSubGroup)->getEndingIndex() <= ((currentCells->getEndingIndex()<<3)+7)
&& (idxSubGroup+1) != tree->getNbCellGroupAtLevel(idxLevel+1)
&& tree->getCellGroup(idxLevel+1, idxSubGroup+1)->getStartingIndex() <= (currentCells->getEndingIndex()<<3)+7 ){
idxSubGroup += 1;
task->dyn_handles[nbSubCellGroups + 1] = handles[idxLevel+1][idxSubGroup];
task->dyn_handles[nbSubCellGroups + 1] = handles_down[idxLevel+1][idxSubGroup];
nbSubCellGroups += 1;
FAssertLF( nbSubCellGroups <= 9 );
}
......@@ -808,7 +837,7 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&p2p_cl_in,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
0);
}
FLOG( timerInBlock.tac() );
......@@ -820,8 +849,8 @@ protected:
starpu_insert_task(&p2p_cl_inout,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_VALUE, &outsideInteractions, sizeof(outsideInteractions),
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_RW, handles[tree->getHeight()][interactionid],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][interactionid],
0);
}
}
......@@ -920,8 +949,8 @@ protected:
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
starpu_insert_task(&l2p_cl,
STARPU_VALUE, &thisptr, sizeof(ThisClass*),
STARPU_R, handles[tree->getHeight()-1][idxGroup],
STARPU_RW, handles[tree->getHeight()][idxGroup],
STARPU_R, handles_down[tree->getHeight()-1][idxGroup],
STARPU_RW, handles_down[tree->getHeight()][idxGroup],
0);
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment