Skip to content

GitLab

Commit 98d24a41 authored by Berenger Bramas's avatar Berenger Bramas
Browse files

update mpi_implcit to rebase against master

parent b57cf433
Expand all Hide whitespace changes
Inline Side-by-side
CMakeLists.txt
View file @ 98d24a41
......@@ -122,6 +122,7 @@ if (MORSE_DISTRIB_DIR OR EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/morse/
CMAKE_DEPENDENT_OPTION(SCALFMM_STARPU_USE_REDUX "Set to ON to enable redux with StarPU" OFF "SCALFMM_USE_STARPU" OFF)
CMAKE_DEPENDENT_OPTION(SCALFMM_STARPU_USE_PRIO "Set to ON to enable priority with StarPU" ON "SCALFMM_USE_STARPU" OFF)
CMAKE_DEPENDENT_OPTION(SCALFMM_STARPU_FORCE_NO_SCHEDULER "Set to ON to disable heteroprio even if supported" OFF "SCALFMM_USE_STARPU" OFF)
CMAKE_DEPENDENT_OPTION(SCALFMM_USE_STARPU_EXTRACT "Set to ON to enable extract with StarPU mpi implicit" ON "SCALFMM_USE_STARPU" OFF)
endif()
message(STATUS "AVANT ${CMAKE_CXX_COMPILER_ID}" )
#
......@@ -460,63 +461,14 @@ if (MORSE_DISTRIB_DIR OR EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/morse/
find_package(FFTW COMPONENTS SIMPLE)
endif()
message(STATUS " SCALFMM USE MKL ")
if( SCALFMM_USE_MKL_AS_BLAS )
unset(FFT_LIBRARIES CACHE)
message(STATUS " SCALFMM USE MKL already defined")
set(FFT_INCLUDES "$ENV{MKLROOT}/include/fftw" CACHE STRING "Set your MKL flags")
if (BLAS_FOUND)
set(FFTW_FOUND ON)
endif()
else(SCALFMM_USE_MKL_AS_BLAS)
# The package can be used with the following COMPONENTS:
# MKL, THREADS|OMP and/or SIMPLE|DOUBLE|LONG|QUAD
# Default is DOUBLE and without THREADS|OMP
find_package(FFTW COMPONENTS MKL) # not REQUIRED
if (FFTW_LIBRARY_DIRS_DEP)
set(FFT_LIBRARIES "-L${FFTW_LIBRARY_DIRS_DEP};" CACHE STRING "Set your MKL flags")
endif()
if (FFTW_LIBRARIES_DEP)
foreach (fft_lib ${FFTW_LIBRARIES_DEP})
set(FFT_LIBRARIES "${FFT_LIBRARIES};${fft_lib};")
endforeach()
endif()
set(FFT_INCLUDES "${FFTW_INCLUDE_DIRS_DEP}" )
if (FFT_LIBRARIES)
set(SCALFMM_LIBRARIES "${SCALFMM_LIBRARIES};${FFT_LIBRARIES}")
endif()
endif(SCALFMM_USE_MKL_AS_BLAS)
else(SCALFMM_USE_MKL_AS_FFTW)
message(STATUS " SCALFMM USE FFTW ")
# The package can be used with the following COMPONENTS:
# MKL, THREADS|OMP and/or SIMPLE|DOUBLE|LONG|QUAD
# Default is DOUBLE and without THREADS|OMP
find_package(FFTW COMPONENTS SIMPLE) # not REQUIRED
#message(FATAL_ERROR ${FFTW_FOUND} ${FFTW_LIBRARY_DIRS_DEP} ${FFTW_LIBRARIES_DEP})
if (FFTW_LIBRARY_DIRS_DEP)
set(FFT_LIBRARIES "-L${FFTW_LIBRARY_DIRS_DEP};" CACHE STRING "Set your FFTW path")
endif()
if (FFTW_LIBRARIES_DEP)
foreach (fft_lib ${FFTW_LIBRARIES_DEP})
set(FFT_LIBRARIES "${FFT_LIBRARIES};${fft_lib};")
endforeach()
endif()
#message(FATAL_ERROR ${FFTW_INCLUDE_DIRS_DEP})
set(FFT_INCLUDES "${FFTW_INCLUDE_DIRS_DEP}" )
if (FFT_LIBRARIES)
set(SCALFMM_LIBRARIES "${SCALFMM_LIBRARIES};${FFT_LIBRARIES}")
endif()
if(FFTW_LIBRARY_DIRS)
# the RPATH to be used when installing
list(APPEND CMAKE_INSTALL_RPATH "${FFTW_LIBRARY_DIRS}")
endif()
if (FFTW_LIBRARY_DIRS_DEP)
set(FFT_LIBRARIES "-L${FFTW_LIBRARY_DIRS_DEP};" CACHE STRING "Set your MKL flags")
endif()
if (FFTW_LIBRARIES_DEP)
foreach (fft_lib ${FFTW_LIBRARIES_DEP})
set(FFT_LIBRARIES "${FFT_LIBRARIES};${fft_lib};")
endforeach()
endif()
set(FFT_INCLUDES "${FFTW_INCLUDE_DIRS_DEP}" )
if (FFT_LIBRARIES)
......
Src/Files/FFmaGenericLoader.hpp
View file @ 98d24a41
......@@ -230,7 +230,7 @@ private:
std::cerr << "File "<< filename<<" not opened! Error: " << strerror(errno) <<std::endl;
std::exit( EXIT_FAILURE);
}
FLOG(FLog::Controller << "Opened file "<< filename << "\n");
std::cout << "Opened file "<< filename << std::endl;
}
public:
......@@ -271,8 +271,8 @@ public:
} else if( filename.find(".fma")!=std::string::npos ) {
binaryFile = false;
} else {
std::cerr << "FFmaGenericLoader: "
<< "Only .fma or .bfma input file are allowed. Got "
std::cout << "FFmaGenericLoader: "
<< "Only .fma or .bfma input file are allowed. Got "
<< filename << "."
<< std::endl;
std::exit ( EXIT_FAILURE) ;
......@@ -350,31 +350,6 @@ public:
unsigned int getDataType(){
return typeData[0]; }
/**
* Fills a particle from the current position in the file.
*
* @param outParticlePositions the position of particle to fill (FPoint<FReal> class)
* @param outPhysicalValue the physical value of particle to fill (FReal)
*/
void fillParticle(FPoint<FReal>*const outParticlePositions){
if(binaryFile){
file->read((char*)(outParticlePositions), sizeof(FReal)*3);
if(otherDataToRead> 0){
file->read((char*)(this->tmpVal), sizeof(FReal)*otherDataToRead);
}
} else {
FReal x,y,z;
(*this->file) >> x >> y >> z;
outParticlePositions->setPosition(x,y,z);
if(otherDataToRead> 0){
for (FSize i = 0 ; i <otherDataToRead; ++i){
(*this->file) >> x ;
}
}
}
}
/**
* Fills a particle from the current position in the file.
*
......@@ -506,25 +481,25 @@ private:
this->readAscciHeader();
}
FLOG(FLog::Controller << " nbParticles: " <<this->nbParticles << std::endl
<< " Box width: " <<this->boxWidth << "\n"
<< " Center: " << this->centerOfBox << "\n");
std::cout << " nbParticles: " <<this->nbParticles << std::endl
<< " Box width: " <<this->boxWidth << std::endl
<< " Center: " << this->centerOfBox << std::endl;
}
void readAscciHeader() {
FLOG(FLog::Controller << " File open in ASCII mode\n");
std::cout << " File open in ASCII mode "<< std::endl ;
FReal x,y,z;
(*this->file) >> typeData[0]>> typeData[1];
FLOG(FLog::Controller << " Datatype "<< typeData[0] << " "<< typeData[1] << "\n");
std::cout << " Datatype "<< typeData[0] << " "<< typeData[1] << std::endl;
(*this->file) >> this->nbParticles >> this->boxWidth >> x >> y >> z;
this->centerOfBox.setPosition(x,y,z);
this->boxWidth *= 2;
otherDataToRead = typeData[1] - (unsigned int)(4);
};
void readBinaryHeader(){
FLOG(FLog::Controller << " File open in binary mode\n");
std::cout << " File open in binary mode "<< std::endl;
file->seekg (std::ios::beg);
file->read((char*)&typeData,2*sizeof(unsigned int));
FLOG(FLog::Controller << " Datatype "<< typeData[0] << " "<< typeData[1] << "\n");
std::cout << " Datatype "<< typeData[0] << " "<< typeData[1] << std::endl;
if(typeData[0] != sizeof(FReal)){
std::cerr << "Size of elements in part file " << typeData[0] << " is different from size of FReal " << sizeof(FReal)<<std::endl;
std::exit( EXIT_FAILURE);
......@@ -610,7 +585,7 @@ public:
this->file->precision(10);
}
else {
std::cerr << "Input file not allowed only .fma or .bfma extensions" <<std::endl;
std::cout << "Input file not allowed only .fma or .bfma extensions" <<std::endl;
std::exit ( EXIT_FAILURE) ;
}
// test if open
......@@ -671,7 +646,7 @@ public:
void writeHeader(const FPoint<FReal> &centerOfBox,const FReal &boxWidth, const FSize &nbParticles, const typePart data) {
unsigned int typeFReal[2] = {sizeof(FReal) , sizeof(typePart) / sizeof(FReal) };
const unsigned int ndata = data.getWriteDataNumber();
FLOG(FLog::Controller <<" WriteHeader: typeFReal: " << typeFReal[0] << " nb Elts: " << typeFReal[1] <<" NData to write "<< ndata<< "\n");
std::cout <<" WriteHeader: typeFReal: " << typeFReal[0] << " nb Elts: " << typeFReal[1] <<" NData to write "<< ndata<< "\n";
if (ndata != typeFReal[1]){
typeFReal[1] = ndata;
}
......@@ -882,16 +857,18 @@ private:
file->seekg (std::ios::beg);
file->write((const char*)typeFReal,2*sizeof(unsigned int));
if(typeFReal[0] != sizeof(FReal)){
std::cerr << "Size of elements in part file " << typeFReal[0] << " is different from size of FReal " << sizeof(FReal)<<std::endl;
std::cout << "Size of elements in part file " << typeFReal[0] << " is different from size of FReal " << sizeof(FReal)<<std::endl;
std::exit( EXIT_FAILURE);
}
else{
file->write( (const char*)&(nbParticles), sizeof(FSize) );
// std::cout << "nbParticles "<< nbParticles<<std::endl;
file->write( (const char*)&(boxWidth) ,sizeof(boxWidth) );
file->write( (const char*)(centerOfBox.getDataValue()),sizeof(FReal)*3);
}
}
};
#endif //FFmaGenericLoader_HPP
#endif //FFmaGenericLoader_HPP
Src/Files/FRandomLoader.hpp
View file @ 98d24a41
......@@ -106,53 +106,6 @@ public:
(getRandom() * boxWidth) + centerOfBox.getY() - boxWidth/2,
(getRandom() * boxWidth) + centerOfBox.getZ() - boxWidth/2);
}
void fillParticleAtMortonIndex(FPoint<FReal>*const inParticlePositions, MortonIndex idx, unsigned int treeHeight){
MortonIndex mask = 0x1LL;
//Largeur de la boite au niveau des feuilles
FReal leafWidth = boxWidth / FReal(1<<(treeHeight-1));
//Décalage par rapport au centre de la moitié de la largeur de la boîte
FReal currentOffset = leafWidth / 2.0;
//Initialise x, y, z au centre de la boîte globale
FReal x, y, z;
x = centerOfBox.getX();
y = centerOfBox.getY();
z = centerOfBox.getZ();
//On va décaler le centre du père vers le centre du fils autant de fois qu'il y a de fils
//Comme ce sont des décalage succesif et plutôt indépendant, on peut commencer par les décalages au niveau des feuilles, ce qui est plus simple
for(unsigned int i = 0; i < treeHeight-1; ++i)
{
bool x_offset, y_offset, z_offset;
//Check le 1er bit qui correspond au z
z_offset = (idx & mask);
idx >>= 1;
//Check le 2nd bit qui correspond au y
y_offset = (idx & mask);
idx >>= 1;
//Check le 3ème bit qui correspond au x
x_offset = (idx & mask);
idx >>= 1;
//Décalage du x
if(x_offset)
x += currentOffset;
else
x -= currentOffset;
//Décalage du y
if(y_offset)
y += currentOffset;
else
y -= currentOffset;
//Décalage du z
if(z_offset)
z += currentOffset;
else
z -= currentOffset;
//On augmente les décallages au fur et à mesure que l'on remonte les étages
currentOffset *= 2;
}
inParticlePositions->setPosition( x, y, z);
}
/** Get a random number between 0 & 1 */
FReal getRandom() const{
......
Src/GroupTree/Core/FGroupTaskStarpuAlgorithm.hpp
View file @ 98d24a41
......@@ -907,7 +907,7 @@ protected:
MortonIndex interactionsIndexes[26];
int interactionsPosition[26];
FTreeCoordinate coord(mindex);
FTreeCoordinate coord(mindex, tree->getHeight()-1);
int counter = coord.getNeighborsIndexes(tree->getHeight(),interactionsIndexes,interactionsPosition);
for(int idxInter = 0 ; idxInter < counter ; ++idxInter){
......@@ -995,7 +995,7 @@ protected:
MortonIndex interactionsIndexes[189];
int interactionsPosition[189];
const FTreeCoordinate coord(mindex);
const FTreeCoordinate coord(mindex, idxLevel);
int counter = coord.getInteractionNeighbors(idxLevel,interactionsIndexes,interactionsPosition);
for(int idxInter = 0 ; idxInter < counter ; ++idxInter){
......
Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp
View file @ 98d24a41
......@@ -15,13 +15,17 @@
#include <vector>
#include <memory>
#ifdef SCALFMM_USE_STARPU_EXTRACT
#include <list>
#endif
#include <omp.h>
#include <starpu.h>
#include <starpu_mpi.h>
#ifdef SCALFMM_USE_STARPU_EXTRACT
#include <algorithm>
#endif
#include "../StarPUUtils/FStarPUUtils.hpp"
#include "../StarPUUtils/FStarPUFmmPriorities.hpp"
#include "../StarPUUtils/FStarPUFmmPrioritiesV2.hpp"
......@@ -94,11 +98,15 @@ protected:
};
std::vector< std::vector< std::vector<BlockInteractions<CellContainerClass>>>> externalInteractionsAllLevel;
#ifdef SCALFMM_USE_STARPU_EXTRACT
std::vector< std::vector< std::vector<std::vector<int>>>> externalInteractionsAllLevelInnerIndexes;
std::vector< std::vector< std::vector<std::vector<int>>>> externalInteractionsAllLevelOuterIndexes;
#endif
std::vector< std::vector<BlockInteractions<ParticleGroupClass>>> externalInteractionsLeafLevel;
#ifdef SCALFMM_USE_STARPU_EXTRACT
std::vector< std::vector<std::vector<int>>> externalInteractionsLeafLevelOuter;
std::vector< std::vector<std::vector<int>>> externalInteractionsLeafLevelInner;
#endif
std::list<const std::vector<OutOfBlockInteraction>*> externalInteractionsLeafLevelOpposite;
OctreeClass*const tree; //< The Tree
......@@ -167,6 +175,7 @@ protected:
int mpi_rank, nproc;
std::vector<std::vector<std::vector<MortonIndex>>> nodeRepartition;
#ifdef SCALFMM_USE_STARPU_EXTRACT
struct ParticleExtractedHandles{
starpu_data_handle_t symb;
size_t size;
......@@ -214,6 +223,7 @@ protected:
starpu_codelet cell_extract_up;
starpu_codelet cell_insert_up;
starpu_codelet cell_insert_up_bis;
#endif
public:
FGroupTaskStarPUImplicitAlgorithm(OctreeClass*const inTree, KernelClass* inKernels, std::vector<MortonIndex>& distributedMortonIndex)
......@@ -836,6 +846,7 @@ protected:
p2p_redux_read.name = "p2p_redux_read";
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
memset(&p2p_extract, 0, sizeof(p2p_extract));
p2p_extract.nbuffers = 2;
p2p_extract.modes[0] = STARPU_R;
......@@ -887,8 +898,10 @@ protected:
cell_insert_up_bis.name = "cell_insert_up_bis";
cell_insert_up_bis.cpu_funcs[0] = ThisClass::InsertCellUpBis;
cell_insert_up_bis.where |= STARPU_CPU;
#endif
}
#ifdef SCALFMM_USE_STARPU_EXTRACT
static void InsertP2P(void *buffers[], void *cl_arg){
ParticleGroupClass containers((unsigned char*)STARPU_VECTOR_GET_PTR(buffers[1]),
STARPU_VECTOR_GET_NX(buffers[1]),
......@@ -982,6 +995,7 @@ protected:
(unsigned char*)STARPU_VECTOR_GET_PTR(buffers[2]),
STARPU_VECTOR_GET_NX(buffers[2]));
}
#endif
void initCodeletMpi(){
memset(&p2p_cl_inout_mpi, 0, sizeof(p2p_cl_inout_mpi));
......@@ -1059,6 +1073,7 @@ protected:
}
particleHandles.clear();
}
#ifdef SCALFMM_USE_STARPU_EXTRACT
for(auto& iter : extractedParticlesBuffer){
starpu_data_unregister(iter.symb);
}
......@@ -1071,6 +1086,7 @@ protected:
for(auto& iter : duplicatedCellBuffer){
starpu_data_unregister(iter.symb);
}
#endif
}
/** Reset the handles array and create new ones to define
......@@ -1148,32 +1164,46 @@ protected:
FLOG( FTic timer; FTic leafTimer; FTic cellTimer; );
// Reset interactions
externalInteractionsAllLevel.clear();
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes.clear();
externalInteractionsAllLevelOuterIndexes.clear();
#endif
externalInteractionsLeafLevel.clear();
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsLeafLevelOuter.clear();
externalInteractionsLeafLevelInner.clear();
#endif
// One per level + leaf level
externalInteractionsAllLevel.resize(tree->getHeight());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes.resize(tree->getHeight());
externalInteractionsAllLevelOuterIndexes.resize(tree->getHeight());
#endif
// First leaf level
{
// We create one big vector per block
externalInteractionsLeafLevel.resize(tree->getNbParticleGroup());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsLeafLevelOuter.resize(tree->getNbParticleGroup());
externalInteractionsLeafLevelInner.resize(tree->getNbParticleGroup());
#endif
for(int idxGroup = 0 ; idxGroup < tree->getNbParticleGroup() ; ++idxGroup){
// Create the vector
ParticleGroupClass* containers = tree->getParticleGroup(idxGroup);
std::vector<BlockInteractions<ParticleGroupClass>>* externalInteractions = &externalInteractionsLeafLevel[idxGroup];
#ifdef SCALFMM_USE_STARPU_EXTRACT
std::vector<std::vector<int>>* externalInteractionsOuter = &externalInteractionsLeafLevelOuter[idxGroup];
std::vector<std::vector<int>>* externalInteractionsInner = &externalInteractionsLeafLevelInner[idxGroup];
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
#pragma omp task default(none) firstprivate(idxGroup, containers, externalInteractions, externalInteractionsOuter, externalInteractionsInner)
#else
#pragma omp task default(none) firstprivate(idxGroup, containers, externalInteractions)
#endif
{ // Can be a task(inout:iterCells)
std::vector<OutOfBlockInteraction> outsideInteractions;
const MortonIndex blockStartIdx = containers->getStartingIndex();
......@@ -1185,7 +1215,7 @@ protected:
MortonIndex interactionsIndexes[26];
int interactionsPosition[26];
FTreeCoordinate coord(mindex, tree->getHeight()-1);
FTreeCoordinate coord(mindex);
int counter = coord.getNeighborsIndexes(tree->getHeight(),interactionsIndexes,interactionsPosition);
for(int idxInter = 0 ; idxInter < counter ; ++idxInter){
......@@ -1245,6 +1275,7 @@ protected:
outsideInteractions.begin() + copyExistingInteraction,
interactions->interactions.begin());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsOuter->emplace_back();
externalInteractionsInner->emplace_back();
......@@ -1260,6 +1291,7 @@ protected:
interactionsOuter->erase(std::unique(interactionsOuter->begin(), interactionsOuter->end()), interactionsOuter->end());
interactionsInner->erase(std::unique(interactionsInner->begin(), interactionsInner->end()), interactionsInner->end());
#endif
}
currentOutInteraction = lastOutInteraction;
......@@ -1272,17 +1304,24 @@ protected:
{
for(int idxLevel = tree->getHeight()-1 ; idxLevel >= 2 ; --idxLevel){
externalInteractionsAllLevel[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
externalInteractionsAllLevelOuterIndexes[idxLevel].resize(tree->getNbCellGroupAtLevel(idxLevel));
#endif
for(int idxGroup = 0 ; idxGroup < tree->getNbCellGroupAtLevel(idxLevel) ; ++idxGroup){
CellContainerClass* currentCells = tree->getCellGroup(idxLevel, idxGroup);
std::vector<BlockInteractions<CellContainerClass>>* externalInteractions = &externalInteractionsAllLevel[idxLevel][idxGroup];
#ifdef SCALFMM_USE_STARPU_EXTRACT
std::vector<std::vector<int>>* externalInteractionsInner = &externalInteractionsAllLevelInnerIndexes[idxLevel][idxGroup];
std::vector<std::vector<int>>* externalInteractionsOuter = &externalInteractionsAllLevelOuterIndexes[idxLevel][idxGroup];
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
#pragma omp task default(none) firstprivate(idxGroup, currentCells, idxLevel, externalInteractions, externalInteractionsInner, externalInteractionsOuter)
#else
#pragma omp task default(none) firstprivate(idxGroup, currentCells, idxLevel, externalInteractions)
#endif
{
std::vector<OutOfBlockInteraction> outsideInteractions;
const MortonIndex blockStartIdx = currentCells->getStartingIndex();
......@@ -1293,7 +1332,7 @@ protected:
MortonIndex interactionsIndexes[189];
int interactionsPosition[189];
const FTreeCoordinate coord(mindex, idxLevel);
const FTreeCoordinate coord(mindex);
int counter = coord.getInteractionNeighbors(idxLevel,interactionsIndexes,interactionsPosition);
for(int idxInter = 0 ; idxInter < counter ; ++idxInter){
......@@ -1354,6 +1393,7 @@ protected:
outsideInteractions.begin() + copyExistingInteraction,
interactions->interactions.begin());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsInner->emplace_back();
std::vector<int>* interactionsInnerIndexes = &externalInteractionsInner->back();
externalInteractionsOuter->emplace_back();
......@@ -1370,6 +1410,7 @@ protected:
FQuickSort<int, int>::QsSequential(interactionsInnerIndexes->data(),int(interactionsInnerIndexes->size()));
interactionsInnerIndexes->erase(std::unique(interactionsInnerIndexes->begin(), interactionsInnerIndexes->end()),
interactionsInnerIndexes->end());
#endif
}
currentOutInteraction = lastOutInteraction;
......@@ -1577,8 +1618,7 @@ protected:
for(int idxInteraction = 0; idxInteraction < int(externalInteractionsAllLevel[idxLevel][idxGroup].size()) ; ++idxInteraction){
const int interactionid = externalInteractionsAllLevel[idxLevel][idxGroup][idxInteraction].otherBlockId;
const std::vector<OutOfBlockInteraction>* outsideInteractions = &externalInteractionsAllLevel[idxLevel][idxGroup][idxInteraction].interactions;
#define M2L_EXTRACT_MPI
#ifdef M2L_EXTRACT_MPI
#ifdef SCALFMM_USE_STARPU_EXTRACT
// On the same node -- do as usual
if(starpu_mpi_data_get_rank(cellHandles[idxLevel][idxGroup].symb) == starpu_mpi_data_get_rank(cellHandles[idxLevel][interactionid].symb)){
#endif
......@@ -1653,7 +1693,8 @@ protected:
#endif
#endif
0);
#ifdef M2L_EXTRACT_MPI
#ifdef SCALFMM_USE_STARPU_EXTRACT
}
else{
{
......@@ -2125,8 +2166,8 @@ protected:
}
else
{
#define P2P_EXTRACT_MPI
#ifdef P2P_EXTRACT_MPI
#ifdef SCALFMM_USE_STARPU_EXTRACT
{
// Extract data from second group for the first one
// That is copy B to B'
......@@ -2281,9 +2322,9 @@ protected:
// Reuse block but just to perform the send
duplicateA.data = (unsigned char*) FAlignedMemory::AllocateBytes<64>(duplicateA.size);// = const_cast<unsigned char*>(tree->getParticleGroup(idxGroup)->getRawBuffer());
}
else{
duplicateA.data = nullptr;
}
else{
duplicateA.data = nullptr;
}
registeringNode = starpu_mpi_data_get_rank(particleHandles[interactionid].down);
where = (registeringNode == mpi_rank) ? STARPU_MAIN_RAM : -1;
......
Src/GroupTree/Core/FGroupTreeBerenger.hpp deleted 100644 → 0
View file @ b57cf433
// Keep in private GIT
#ifndef FGROUPTREE_HPP
#define FGROUPTREE_HPP
#include <vector>
#include <functional>
#include "../../Utils/FAssert.hpp"
#include "../../Utils/FPoint.hpp"
#include "../../Utils/FQuickSort.hpp"
#include "../../Containers/FTreeCoordinate.hpp"
#include "../../Containers/FCoordinateComputer.hpp"
#include "FGroupOfCells.hpp"
#include "FGroupOfParticles.hpp"
#include "FGroupAttachedLeaf.hpp"
template <class FReal, class CompositeCellClass, class SymboleCellClass, class PoleCellClass, class LocalCellClass,
class GroupAttachedLeafClass, unsigned NbSymbAttributes, unsigned NbAttributesPerParticle, class AttributeClass = FReal>
class FGroupTreeBerenger {
public:
typedef GroupAttachedLeafClass BasicAttachedClass;
typedef FGroupOfParticles<FReal, NbSymbAttributes, NbAttributesPerParticle,AttributeClass> ParticleGroupClass;
typedef FGroupOfCells<CompositeCellClass, SymboleCellClass, PoleCellClass, LocalCellClass> CellGroupClass;
protected:
//< height of the tree (1 => only the root)
const int treeHeight;
//< max number of cells in a block
const int nbElementsPerBlock;
//< all the blocks of the tree
std::vector<CellGroupClass*>* cellBlocksPerLevel;
//< all the blocks of leaves
std::vector<ParticleGroupClass*> particleBlocks;
//< the space system center
const FPoint<FReal> boxCenter;
//< the space system corner (used to compute morton index)
const FPoint<FReal> boxCorner;
//< the space system width
const FReal boxWidth;
//< the width of a box at width level
const FReal boxWidthAtLeafLevel;
public:
typedef typename std::vector<CellGroupClass*>::iterator CellGroupIterator;
typedef typename std::vector<CellGroupClass*>::const_iterator CellGroupConstIterator;
typedef typename std::vector<ParticleGroupClass*>::iterator ParticleGroupIterator;
typedef typename std::vector<ParticleGroupClass*>::const_iterator ParticleGroupConstIterator;
/**
* This constructor create a group tree from a particle container index.
* The morton index are computed and the particles are sorted in a first stage.
* Then the leaf level is done.
* Finally the other leve are proceed one after the other.
* It should be easy to make it parallel using for and tasks.
* If no limite give inLeftLimite = -1
*/
template<class ParticleContainer>
FGroupTreeBerenger(const int inTreeHeight, const FReal inBoxWidth, const FPoint<FReal>& inBoxCenter,
const int inNbElementsPerBlock, ParticleContainer* inParticlesContainer,
const bool particlesAreSorted, std::vector<MortonIndex> const& distributedMortonIndex, MortonIndex inLeftLimite = -1):
treeHeight(inTreeHeight),nbElementsPerBlock(inNbElementsPerBlock),cellBlocksPerLevel(nullptr),
boxCenter(inBoxCenter), boxCorner(inBoxCenter,-(inBoxWidth/2)), boxWidth(inBoxWidth),
boxWidthAtLeafLevel(inBoxWidth/FReal(1<<(inTreeHeight-1))){