Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
solverstack
ScalFMM
Commits
98d24a41
Commit
98d24a41
authored
Mar 14, 2017
by
Berenger Bramas
Browse files
update mpi_implcit to rebase against master
parent
b57cf433
Changes
17
Expand all
Hide whitespace changes
Inline
Side-by-side
CMakeLists.txt
View file @
98d24a41
...
...
@@ -122,6 +122,7 @@ if (MORSE_DISTRIB_DIR OR EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/morse/
CMAKE_DEPENDENT_OPTION
(
SCALFMM_STARPU_USE_REDUX
"Set to ON to enable redux with StarPU"
OFF
"SCALFMM_USE_STARPU"
OFF
)
CMAKE_DEPENDENT_OPTION
(
SCALFMM_STARPU_USE_PRIO
"Set to ON to enable priority with StarPU"
ON
"SCALFMM_USE_STARPU"
OFF
)
CMAKE_DEPENDENT_OPTION
(
SCALFMM_STARPU_FORCE_NO_SCHEDULER
"Set to ON to disable heteroprio even if supported"
OFF
"SCALFMM_USE_STARPU"
OFF
)
CMAKE_DEPENDENT_OPTION
(
SCALFMM_USE_STARPU_EXTRACT
"Set to ON to enable extract with StarPU mpi implicit"
ON
"SCALFMM_USE_STARPU"
OFF
)
endif
()
message
(
STATUS
"AVANT
${
CMAKE_CXX_COMPILER_ID
}
"
)
#
...
...
@@ -460,63 +461,14 @@ if (MORSE_DISTRIB_DIR OR EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/morse/
find_package
(
FFTW COMPONENTS SIMPLE
)
endif
()
message
(
STATUS
" SCALFMM USE MKL "
)
if
(
SCALFMM_USE_MKL_AS_BLAS
)
unset
(
FFT_LIBRARIES CACHE
)
message
(
STATUS
" SCALFMM USE MKL already defined"
)
set
(
FFT_INCLUDES
"$ENV{MKLROOT}/include/fftw"
CACHE STRING
"Set your MKL flags"
)
if
(
BLAS_FOUND
)
set
(
FFTW_FOUND ON
)
endif
()
else
(
SCALFMM_USE_MKL_AS_BLAS
)
# The package can be used with the following COMPONENTS:
# MKL, THREADS|OMP and/or SIMPLE|DOUBLE|LONG|QUAD
# Default is DOUBLE and without THREADS|OMP
find_package
(
FFTW COMPONENTS MKL
)
# not REQUIRED
if
(
FFTW_LIBRARY_DIRS_DEP
)
set
(
FFT_LIBRARIES
"-L
${
FFTW_LIBRARY_DIRS_DEP
}
;"
CACHE STRING
"Set your MKL flags"
)
endif
()
if
(
FFTW_LIBRARIES_DEP
)
foreach
(
fft_lib
${
FFTW_LIBRARIES_DEP
}
)
set
(
FFT_LIBRARIES
"
${
FFT_LIBRARIES
}
;
${
fft_lib
}
;"
)
endforeach
()
endif
()
set
(
FFT_INCLUDES
"
${
FFTW_INCLUDE_DIRS_DEP
}
"
)
if
(
FFT_LIBRARIES
)
set
(
SCALFMM_LIBRARIES
"
${
SCALFMM_LIBRARIES
}
;
${
FFT_LIBRARIES
}
"
)
endif
()
endif
(
SCALFMM_USE_MKL_AS_BLAS
)
else
(
SCALFMM_USE_MKL_AS_FFTW
)
message
(
STATUS
" SCALFMM USE FFTW "
)
# The package can be used with the following COMPONENTS:
# MKL, THREADS|OMP and/or SIMPLE|DOUBLE|LONG|QUAD
# Default is DOUBLE and without THREADS|OMP
find_package
(
FFTW COMPONENTS SIMPLE
)
# not REQUIRED
#message(FATAL_ERROR ${FFTW_FOUND} ${FFTW_LIBRARY_DIRS_DEP} ${FFTW_LIBRARIES_DEP})
if
(
FFTW_LIBRARY_DIRS_DEP
)
set
(
FFT_LIBRARIES
"-L
${
FFTW_LIBRARY_DIRS_DEP
}
;"
CACHE STRING
"Set your FFTW path"
)
endif
()
if
(
FFTW_LIBRARIES_DEP
)
foreach
(
fft_lib
${
FFTW_LIBRARIES_DEP
}
)
set
(
FFT_LIBRARIES
"
${
FFT_LIBRARIES
}
;
${
fft_lib
}
;"
)
endforeach
()
endif
()
#message(FATAL_ERROR ${FFTW_INCLUDE_DIRS_DEP})
set
(
FFT_INCLUDES
"
${
FFTW_INCLUDE_DIRS_DEP
}
"
)
if
(
FFT_LIBRARIES
)
set
(
SCALFMM_LIBRARIES
"
${
SCALFMM_LIBRARIES
}
;
${
FFT_LIBRARIES
}
"
)
endif
()
if
(
FFTW_LIBRARY_DIRS
)
# the RPATH to be used when installing
list
(
APPEND CMAKE_INSTALL_RPATH
"
${
FFTW_LIBRARY_DIRS
}
"
)
endif
()
if
(
FFTW_LIBRARY_DIRS_DEP
)
set
(
FFT_LIBRARIES
"-L
${
FFTW_LIBRARY_DIRS_DEP
}
;"
CACHE STRING
"Set your MKL flags"
)
endif
()
if
(
FFTW_LIBRARIES_DEP
)
foreach
(
fft_lib
${
FFTW_LIBRARIES_DEP
}
)
set
(
FFT_LIBRARIES
"
${
FFT_LIBRARIES
}
;
${
fft_lib
}
;"
)
endforeach
()
endif
()
set
(
FFT_INCLUDES
"
${
FFTW_INCLUDE_DIRS_DEP
}
"
)
if
(
FFT_LIBRARIES
)
...
...
Src/Files/FFmaGenericLoader.hpp
View file @
98d24a41
...
...
@@ -230,7 +230,7 @@ private:
std
::
cerr
<<
"File "
<<
filename
<<
" not opened! Error: "
<<
strerror
(
errno
)
<<
std
::
endl
;
std
::
exit
(
EXIT_FAILURE
);
}
FLOG
(
FLog
::
Controller
<<
"Opened file "
<<
filename
<<
"
\n
"
)
;
std
::
cout
<<
"Opened file "
<<
filename
<<
std
::
endl
;
}
public:
...
...
@@ -271,8 +271,8 @@ public:
}
else
if
(
filename
.
find
(
".fma"
)
!=
std
::
string
::
npos
)
{
binaryFile
=
false
;
}
else
{
std
::
c
err
<<
"FFmaGenericLoader: "
<<
"Only .fma or .bfma input file are allowed. Got "
std
::
c
out
<<
"FFmaGenericLoader: "
<<
"Only .fma or .bfma input file are allowed. Got "
<<
filename
<<
"."
<<
std
::
endl
;
std
::
exit
(
EXIT_FAILURE
)
;
...
...
@@ -350,31 +350,6 @@ public:
unsigned
int
getDataType
(){
return
typeData
[
0
];
}
/**
* Fills a particle from the current position in the file.
*
* @param outParticlePositions the position of particle to fill (FPoint<FReal> class)
* @param outPhysicalValue the physical value of particle to fill (FReal)
*/
void
fillParticle
(
FPoint
<
FReal
>*
const
outParticlePositions
){
if
(
binaryFile
){
file
->
read
((
char
*
)(
outParticlePositions
),
sizeof
(
FReal
)
*
3
);
if
(
otherDataToRead
>
0
){
file
->
read
((
char
*
)(
this
->
tmpVal
),
sizeof
(
FReal
)
*
otherDataToRead
);
}
}
else
{
FReal
x
,
y
,
z
;
(
*
this
->
file
)
>>
x
>>
y
>>
z
;
outParticlePositions
->
setPosition
(
x
,
y
,
z
);
if
(
otherDataToRead
>
0
){
for
(
FSize
i
=
0
;
i
<
otherDataToRead
;
++
i
){
(
*
this
->
file
)
>>
x
;
}
}
}
}
/**
* Fills a particle from the current position in the file.
*
...
...
@@ -506,25 +481,25 @@ private:
this
->
readAscciHeader
();
}
FLOG
(
FLog
::
Controller
<<
" nbParticles: "
<<
this
->
nbParticles
<<
std
::
endl
<<
" Box width: "
<<
this
->
boxWidth
<<
"
\n
"
<<
" Center: "
<<
this
->
centerOfBox
<<
"
\n
"
)
;
std
::
cout
<<
" nbParticles: "
<<
this
->
nbParticles
<<
std
::
endl
<<
" Box width: "
<<
this
->
boxWidth
<<
std
::
endl
<<
" Center: "
<<
this
->
centerOfBox
<<
std
::
endl
;
}
void
readAscciHeader
()
{
FLOG
(
FLog
::
Controller
<<
" File open in ASCII mode
\n
"
)
;
std
::
cout
<<
" File open in ASCII mode
"
<<
std
::
endl
;
FReal
x
,
y
,
z
;
(
*
this
->
file
)
>>
typeData
[
0
]
>>
typeData
[
1
];
FLOG
(
FLog
::
Controller
<<
" Datatype "
<<
typeData
[
0
]
<<
" "
<<
typeData
[
1
]
<<
"
\n
"
)
;
std
::
cout
<<
" Datatype "
<<
typeData
[
0
]
<<
" "
<<
typeData
[
1
]
<<
std
::
endl
;
(
*
this
->
file
)
>>
this
->
nbParticles
>>
this
->
boxWidth
>>
x
>>
y
>>
z
;
this
->
centerOfBox
.
setPosition
(
x
,
y
,
z
);
this
->
boxWidth
*=
2
;
otherDataToRead
=
typeData
[
1
]
-
(
unsigned
int
)(
4
);
};
void
readBinaryHeader
(){
FLOG
(
FLog
::
Controller
<<
" File open in binary mode
\n
"
)
;
std
::
cout
<<
" File open in binary mode
"
<<
std
::
endl
;
file
->
seekg
(
std
::
ios
::
beg
);
file
->
read
((
char
*
)
&
typeData
,
2
*
sizeof
(
unsigned
int
));
FLOG
(
FLog
::
Controller
<<
" Datatype "
<<
typeData
[
0
]
<<
" "
<<
typeData
[
1
]
<<
"
\n
"
)
;
std
::
cout
<<
" Datatype "
<<
typeData
[
0
]
<<
" "
<<
typeData
[
1
]
<<
std
::
endl
;
if
(
typeData
[
0
]
!=
sizeof
(
FReal
)){
std
::
cerr
<<
"Size of elements in part file "
<<
typeData
[
0
]
<<
" is different from size of FReal "
<<
sizeof
(
FReal
)
<<
std
::
endl
;
std
::
exit
(
EXIT_FAILURE
);
...
...
@@ -610,7 +585,7 @@ public:
this
->
file
->
precision
(
10
);
}
else
{
std
::
c
err
<<
"Input file not allowed only .fma or .bfma extensions"
<<
std
::
endl
;
std
::
c
out
<<
"Input file not allowed only .fma or .bfma extensions"
<<
std
::
endl
;
std
::
exit
(
EXIT_FAILURE
)
;
}
// test if open
...
...
@@ -671,7 +646,7 @@ public:
void
writeHeader
(
const
FPoint
<
FReal
>
&
centerOfBox
,
const
FReal
&
boxWidth
,
const
FSize
&
nbParticles
,
const
typePart
data
)
{
unsigned
int
typeFReal
[
2
]
=
{
sizeof
(
FReal
)
,
sizeof
(
typePart
)
/
sizeof
(
FReal
)
};
const
unsigned
int
ndata
=
data
.
getWriteDataNumber
();
FLOG
(
FLog
::
Controller
<<
" WriteHeader: typeFReal: "
<<
typeFReal
[
0
]
<<
" nb Elts: "
<<
typeFReal
[
1
]
<<
" NData to write "
<<
ndata
<<
"
\n
"
)
;
std
::
cout
<<
" WriteHeader: typeFReal: "
<<
typeFReal
[
0
]
<<
" nb Elts: "
<<
typeFReal
[
1
]
<<
" NData to write "
<<
ndata
<<
"
\n
"
;
if
(
ndata
!=
typeFReal
[
1
]){
typeFReal
[
1
]
=
ndata
;
}
...
...
@@ -882,16 +857,18 @@ private:
file
->
seekg
(
std
::
ios
::
beg
);
file
->
write
((
const
char
*
)
typeFReal
,
2
*
sizeof
(
unsigned
int
));
if
(
typeFReal
[
0
]
!=
sizeof
(
FReal
)){
std
::
c
err
<<
"Size of elements in part file "
<<
typeFReal
[
0
]
<<
" is different from size of FReal "
<<
sizeof
(
FReal
)
<<
std
::
endl
;
std
::
c
out
<<
"Size of elements in part file "
<<
typeFReal
[
0
]
<<
" is different from size of FReal "
<<
sizeof
(
FReal
)
<<
std
::
endl
;
std
::
exit
(
EXIT_FAILURE
);
}
else
{
file
->
write
(
(
const
char
*
)
&
(
nbParticles
),
sizeof
(
FSize
)
);
// std::cout << "nbParticles "<< nbParticles<<std::endl;
file
->
write
(
(
const
char
*
)
&
(
boxWidth
)
,
sizeof
(
boxWidth
)
);
file
->
write
(
(
const
char
*
)(
centerOfBox
.
getDataValue
()),
sizeof
(
FReal
)
*
3
);
}
}
};
#endif //FFmaGenericLoader_HPP
#endif //FFmaGenericLoader_HPP
Src/Files/FRandomLoader.hpp
View file @
98d24a41
...
...
@@ -106,53 +106,6 @@ public:
(
getRandom
()
*
boxWidth
)
+
centerOfBox
.
getY
()
-
boxWidth
/
2
,
(
getRandom
()
*
boxWidth
)
+
centerOfBox
.
getZ
()
-
boxWidth
/
2
);
}
void
fillParticleAtMortonIndex
(
FPoint
<
FReal
>*
const
inParticlePositions
,
MortonIndex
idx
,
unsigned
int
treeHeight
){
MortonIndex
mask
=
0x1LL
;
//Largeur de la boite au niveau des feuilles
FReal
leafWidth
=
boxWidth
/
FReal
(
1
<<
(
treeHeight
-
1
));
//Décalage par rapport au centre de la moitié de la largeur de la boîte
FReal
currentOffset
=
leafWidth
/
2.0
;
//Initialise x, y, z au centre de la boîte globale
FReal
x
,
y
,
z
;
x
=
centerOfBox
.
getX
();
y
=
centerOfBox
.
getY
();
z
=
centerOfBox
.
getZ
();
//On va décaler le centre du père vers le centre du fils autant de fois qu'il y a de fils
//Comme ce sont des décalage succesif et plutôt indépendant, on peut commencer par les décalages au niveau des feuilles, ce qui est plus simple
for
(
unsigned
int
i
=
0
;
i
<
treeHeight
-
1
;
++
i
)
{
bool
x_offset
,
y_offset
,
z_offset
;
//Check le 1er bit qui correspond au z
z_offset
=
(
idx
&
mask
);
idx
>>=
1
;
//Check le 2nd bit qui correspond au y
y_offset
=
(
idx
&
mask
);
idx
>>=
1
;
//Check le 3ème bit qui correspond au x
x_offset
=
(
idx
&
mask
);
idx
>>=
1
;
//Décalage du x
if
(
x_offset
)
x
+=
currentOffset
;
else
x
-=
currentOffset
;
//Décalage du y
if
(
y_offset
)
y
+=
currentOffset
;
else
y
-=
currentOffset
;
//Décalage du z
if
(
z_offset
)
z
+=
currentOffset
;
else
z
-=
currentOffset
;
//On augmente les décallages au fur et à mesure que l'on remonte les étages
currentOffset
*=
2
;
}
inParticlePositions
->
setPosition
(
x
,
y
,
z
);
}
/** Get a random number between 0 & 1 */
FReal
getRandom
()
const
{
...
...
Src/GroupTree/Core/FGroupTaskStarpuAlgorithm.hpp
View file @
98d24a41
...
...
@@ -907,7 +907,7 @@ protected:
MortonIndex
interactionsIndexes
[
26
];
int
interactionsPosition
[
26
];
FTreeCoordinate
coord
(
mindex
);
FTreeCoordinate
coord
(
mindex
,
tree
->
getHeight
()
-
1
);
int
counter
=
coord
.
getNeighborsIndexes
(
tree
->
getHeight
(),
interactionsIndexes
,
interactionsPosition
);
for
(
int
idxInter
=
0
;
idxInter
<
counter
;
++
idxInter
){
...
...
@@ -995,7 +995,7 @@ protected:
MortonIndex
interactionsIndexes
[
189
];
int
interactionsPosition
[
189
];
const
FTreeCoordinate
coord
(
mindex
);
const
FTreeCoordinate
coord
(
mindex
,
idxLevel
);
int
counter
=
coord
.
getInteractionNeighbors
(
idxLevel
,
interactionsIndexes
,
interactionsPosition
);
for
(
int
idxInter
=
0
;
idxInter
<
counter
;
++
idxInter
){
...
...
Src/GroupTree/Core/FGroupTaskStarpuImplicitAlgorithm.hpp
View file @
98d24a41
...
...
@@ -15,13 +15,17 @@
#include <vector>
#include <memory>
#ifdef SCALFMM_USE_STARPU_EXTRACT
#include <list>
#endif
#include <omp.h>
#include <starpu.h>
#include <starpu_mpi.h>
#ifdef SCALFMM_USE_STARPU_EXTRACT
#include <algorithm>
#endif
#include "../StarPUUtils/FStarPUUtils.hpp"
#include "../StarPUUtils/FStarPUFmmPriorities.hpp"
#include "../StarPUUtils/FStarPUFmmPrioritiesV2.hpp"
...
...
@@ -94,11 +98,15 @@ protected:
};
std
::
vector
<
std
::
vector
<
std
::
vector
<
BlockInteractions
<
CellContainerClass
>>>>
externalInteractionsAllLevel
;
#ifdef SCALFMM_USE_STARPU_EXTRACT
std
::
vector
<
std
::
vector
<
std
::
vector
<
std
::
vector
<
int
>>>>
externalInteractionsAllLevelInnerIndexes
;
std
::
vector
<
std
::
vector
<
std
::
vector
<
std
::
vector
<
int
>>>>
externalInteractionsAllLevelOuterIndexes
;
#endif
std
::
vector
<
std
::
vector
<
BlockInteractions
<
ParticleGroupClass
>>>
externalInteractionsLeafLevel
;
#ifdef SCALFMM_USE_STARPU_EXTRACT
std
::
vector
<
std
::
vector
<
std
::
vector
<
int
>>>
externalInteractionsLeafLevelOuter
;
std
::
vector
<
std
::
vector
<
std
::
vector
<
int
>>>
externalInteractionsLeafLevelInner
;
#endif
std
::
list
<
const
std
::
vector
<
OutOfBlockInteraction
>*>
externalInteractionsLeafLevelOpposite
;
OctreeClass
*
const
tree
;
//< The Tree
...
...
@@ -167,6 +175,7 @@ protected:
int
mpi_rank
,
nproc
;
std
::
vector
<
std
::
vector
<
std
::
vector
<
MortonIndex
>>>
nodeRepartition
;
#ifdef SCALFMM_USE_STARPU_EXTRACT
struct
ParticleExtractedHandles
{
starpu_data_handle_t
symb
;
size_t
size
;
...
...
@@ -214,6 +223,7 @@ protected:
starpu_codelet
cell_extract_up
;
starpu_codelet
cell_insert_up
;
starpu_codelet
cell_insert_up_bis
;
#endif
public:
FGroupTaskStarPUImplicitAlgorithm
(
OctreeClass
*
const
inTree
,
KernelClass
*
inKernels
,
std
::
vector
<
MortonIndex
>&
distributedMortonIndex
)
...
...
@@ -836,6 +846,7 @@ protected:
p2p_redux_read
.
name
=
"p2p_redux_read"
;
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
memset
(
&
p2p_extract
,
0
,
sizeof
(
p2p_extract
));
p2p_extract
.
nbuffers
=
2
;
p2p_extract
.
modes
[
0
]
=
STARPU_R
;
...
...
@@ -887,8 +898,10 @@ protected:
cell_insert_up_bis
.
name
=
"cell_insert_up_bis"
;
cell_insert_up_bis
.
cpu_funcs
[
0
]
=
ThisClass
::
InsertCellUpBis
;
cell_insert_up_bis
.
where
|=
STARPU_CPU
;
#endif
}
#ifdef SCALFMM_USE_STARPU_EXTRACT
static
void
InsertP2P
(
void
*
buffers
[],
void
*
cl_arg
){
ParticleGroupClass
containers
((
unsigned
char
*
)
STARPU_VECTOR_GET_PTR
(
buffers
[
1
]),
STARPU_VECTOR_GET_NX
(
buffers
[
1
]),
...
...
@@ -982,6 +995,7 @@ protected:
(
unsigned
char
*
)
STARPU_VECTOR_GET_PTR
(
buffers
[
2
]),
STARPU_VECTOR_GET_NX
(
buffers
[
2
]));
}
#endif
void
initCodeletMpi
(){
memset
(
&
p2p_cl_inout_mpi
,
0
,
sizeof
(
p2p_cl_inout_mpi
));
...
...
@@ -1059,6 +1073,7 @@ protected:
}
particleHandles
.
clear
();
}
#ifdef SCALFMM_USE_STARPU_EXTRACT
for
(
auto
&
iter
:
extractedParticlesBuffer
){
starpu_data_unregister
(
iter
.
symb
);
}
...
...
@@ -1071,6 +1086,7 @@ protected:
for
(
auto
&
iter
:
duplicatedCellBuffer
){
starpu_data_unregister
(
iter
.
symb
);
}
#endif
}
/** Reset the handles array and create new ones to define
...
...
@@ -1148,32 +1164,46 @@ protected:
FLOG
(
FTic
timer
;
FTic
leafTimer
;
FTic
cellTimer
;
);
// Reset interactions
externalInteractionsAllLevel
.
clear
();
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes
.
clear
();
externalInteractionsAllLevelOuterIndexes
.
clear
();
#endif
externalInteractionsLeafLevel
.
clear
();
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsLeafLevelOuter
.
clear
();
externalInteractionsLeafLevelInner
.
clear
();
#endif
// One per level + leaf level
externalInteractionsAllLevel
.
resize
(
tree
->
getHeight
());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes
.
resize
(
tree
->
getHeight
());
externalInteractionsAllLevelOuterIndexes
.
resize
(
tree
->
getHeight
());
#endif
// First leaf level
{
// We create one big vector per block
externalInteractionsLeafLevel
.
resize
(
tree
->
getNbParticleGroup
());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsLeafLevelOuter
.
resize
(
tree
->
getNbParticleGroup
());
externalInteractionsLeafLevelInner
.
resize
(
tree
->
getNbParticleGroup
());
#endif
for
(
int
idxGroup
=
0
;
idxGroup
<
tree
->
getNbParticleGroup
()
;
++
idxGroup
){
// Create the vector
ParticleGroupClass
*
containers
=
tree
->
getParticleGroup
(
idxGroup
);
std
::
vector
<
BlockInteractions
<
ParticleGroupClass
>>*
externalInteractions
=
&
externalInteractionsLeafLevel
[
idxGroup
];
#ifdef SCALFMM_USE_STARPU_EXTRACT
std
::
vector
<
std
::
vector
<
int
>>*
externalInteractionsOuter
=
&
externalInteractionsLeafLevelOuter
[
idxGroup
];
std
::
vector
<
std
::
vector
<
int
>>*
externalInteractionsInner
=
&
externalInteractionsLeafLevelInner
[
idxGroup
];
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
#pragma omp task default(none) firstprivate(idxGroup, containers, externalInteractions, externalInteractionsOuter, externalInteractionsInner)
#else
#pragma omp task default(none) firstprivate(idxGroup, containers, externalInteractions)
#endif
{
// Can be a task(inout:iterCells)
std
::
vector
<
OutOfBlockInteraction
>
outsideInteractions
;
const
MortonIndex
blockStartIdx
=
containers
->
getStartingIndex
();
...
...
@@ -1185,7 +1215,7 @@ protected:
MortonIndex
interactionsIndexes
[
26
];
int
interactionsPosition
[
26
];
FTreeCoordinate
coord
(
mindex
,
tree
->
getHeight
()
-
1
);
FTreeCoordinate
coord
(
mindex
);
int
counter
=
coord
.
getNeighborsIndexes
(
tree
->
getHeight
(),
interactionsIndexes
,
interactionsPosition
);
for
(
int
idxInter
=
0
;
idxInter
<
counter
;
++
idxInter
){
...
...
@@ -1245,6 +1275,7 @@ protected:
outsideInteractions
.
begin
()
+
copyExistingInteraction
,
interactions
->
interactions
.
begin
());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsOuter
->
emplace_back
();
externalInteractionsInner
->
emplace_back
();
...
...
@@ -1260,6 +1291,7 @@ protected:
interactionsOuter
->
erase
(
std
::
unique
(
interactionsOuter
->
begin
(),
interactionsOuter
->
end
()),
interactionsOuter
->
end
());
interactionsInner
->
erase
(
std
::
unique
(
interactionsInner
->
begin
(),
interactionsInner
->
end
()),
interactionsInner
->
end
());
#endif
}
currentOutInteraction
=
lastOutInteraction
;
...
...
@@ -1272,17 +1304,24 @@ protected:
{
for
(
int
idxLevel
=
tree
->
getHeight
()
-
1
;
idxLevel
>=
2
;
--
idxLevel
){
externalInteractionsAllLevel
[
idxLevel
].
resize
(
tree
->
getNbCellGroupAtLevel
(
idxLevel
));
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsAllLevelInnerIndexes
[
idxLevel
].
resize
(
tree
->
getNbCellGroupAtLevel
(
idxLevel
));
externalInteractionsAllLevelOuterIndexes
[
idxLevel
].
resize
(
tree
->
getNbCellGroupAtLevel
(
idxLevel
));
#endif
for
(
int
idxGroup
=
0
;
idxGroup
<
tree
->
getNbCellGroupAtLevel
(
idxLevel
)
;
++
idxGroup
){
CellContainerClass
*
currentCells
=
tree
->
getCellGroup
(
idxLevel
,
idxGroup
);
std
::
vector
<
BlockInteractions
<
CellContainerClass
>>*
externalInteractions
=
&
externalInteractionsAllLevel
[
idxLevel
][
idxGroup
];
#ifdef SCALFMM_USE_STARPU_EXTRACT
std
::
vector
<
std
::
vector
<
int
>>*
externalInteractionsInner
=
&
externalInteractionsAllLevelInnerIndexes
[
idxLevel
][
idxGroup
];
std
::
vector
<
std
::
vector
<
int
>>*
externalInteractionsOuter
=
&
externalInteractionsAllLevelOuterIndexes
[
idxLevel
][
idxGroup
];
#endif
#ifdef SCALFMM_USE_STARPU_EXTRACT
#pragma omp task default(none) firstprivate(idxGroup, currentCells, idxLevel, externalInteractions, externalInteractionsInner, externalInteractionsOuter)
#else
#pragma omp task default(none) firstprivate(idxGroup, currentCells, idxLevel, externalInteractions)
#endif
{
std
::
vector
<
OutOfBlockInteraction
>
outsideInteractions
;
const
MortonIndex
blockStartIdx
=
currentCells
->
getStartingIndex
();
...
...
@@ -1293,7 +1332,7 @@ protected:
MortonIndex
interactionsIndexes
[
189
];
int
interactionsPosition
[
189
];
const
FTreeCoordinate
coord
(
mindex
,
idxLevel
);
const
FTreeCoordinate
coord
(
mindex
);
int
counter
=
coord
.
getInteractionNeighbors
(
idxLevel
,
interactionsIndexes
,
interactionsPosition
);
for
(
int
idxInter
=
0
;
idxInter
<
counter
;
++
idxInter
){
...
...
@@ -1354,6 +1393,7 @@ protected:
outsideInteractions
.
begin
()
+
copyExistingInteraction
,
interactions
->
interactions
.
begin
());
#ifdef SCALFMM_USE_STARPU_EXTRACT
externalInteractionsInner
->
emplace_back
();
std
::
vector
<
int
>*
interactionsInnerIndexes
=
&
externalInteractionsInner
->
back
();
externalInteractionsOuter
->
emplace_back
();
...
...
@@ -1370,6 +1410,7 @@ protected:
FQuickSort
<
int
,
int
>::
QsSequential
(
interactionsInnerIndexes
->
data
(),
int
(
interactionsInnerIndexes
->
size
()));
interactionsInnerIndexes
->
erase
(
std
::
unique
(
interactionsInnerIndexes
->
begin
(),
interactionsInnerIndexes
->
end
()),
interactionsInnerIndexes
->
end
());
#endif
}
currentOutInteraction
=
lastOutInteraction
;
...
...
@@ -1577,8 +1618,7 @@ protected:
for
(
int
idxInteraction
=
0
;
idxInteraction
<
int
(
externalInteractionsAllLevel
[
idxLevel
][
idxGroup
].
size
())
;
++
idxInteraction
){
const
int
interactionid
=
externalInteractionsAllLevel
[
idxLevel
][
idxGroup
][
idxInteraction
].
otherBlockId
;
const
std
::
vector
<
OutOfBlockInteraction
>*
outsideInteractions
=
&
externalInteractionsAllLevel
[
idxLevel
][
idxGroup
][
idxInteraction
].
interactions
;
#define M2L_EXTRACT_MPI
#ifdef M2L_EXTRACT_MPI
#ifdef SCALFMM_USE_STARPU_EXTRACT
// On the same node -- do as usual
if
(
starpu_mpi_data_get_rank
(
cellHandles
[
idxLevel
][
idxGroup
].
symb
)
==
starpu_mpi_data_get_rank
(
cellHandles
[
idxLevel
][
interactionid
].
symb
)){
#endif
...
...
@@ -1653,7 +1693,8 @@ protected:
#endif
#endif
0
);
#ifdef M2L_EXTRACT_MPI
#ifdef SCALFMM_USE_STARPU_EXTRACT
}
else
{
{
...
...
@@ -2125,8 +2166,8 @@ protected:
}
else
{
#define P2P_EXTRACT_MPI
#ifdef
P2P
_EXTRACT
_MPI
#ifdef
SCALFMM_USE_STARPU
_EXTRACT
{
// Extract data from second group for the first one
// That is copy B to B'
...
...
@@ -2281,9 +2322,9 @@ protected:
// Reuse block but just to perform the send
duplicateA
.
data
=
(
unsigned
char
*
)
FAlignedMemory
::
AllocateBytes
<
64
>
(
duplicateA
.
size
);
// = const_cast<unsigned char*>(tree->getParticleGroup(idxGroup)->getRawBuffer());
}
else
{
duplicateA
.
data
=
nullptr
;
}
else
{
duplicateA
.
data
=
nullptr
;
}
registeringNode
=
starpu_mpi_data_get_rank
(
particleHandles
[
interactionid
].
down
);
where
=
(
registeringNode
==
mpi_rank
)
?
STARPU_MAIN_RAM
:
-
1
;
...
...
Src/GroupTree/Core/FGroupTaskStarpuMpiAlgorithm.hpp
View file @
98d24a41
This diff is collapsed.
Click to expand it.
Src/GroupTree/Core/FGroupTreeBerenger.hpp
deleted
100644 → 0
View file @
b57cf433
// Keep in private GIT
#ifndef FGROUPTREE_HPP
#define FGROUPTREE_HPP
#include <vector>
#include <functional>
#include "../../Utils/FAssert.hpp"
#include "../../Utils/FPoint.hpp"
#include "../../Utils/FQuickSort.hpp"
#include "../../Containers/FTreeCoordinate.hpp"
#include "../../Containers/FCoordinateComputer.hpp"
#include "FGroupOfCells.hpp"
#include "FGroupOfParticles.hpp"
#include "FGroupAttachedLeaf.hpp"
template
<
class
FReal
,
class
CompositeCellClass
,
class
SymboleCellClass
,
class
PoleCellClass
,
class
LocalCellClass
,
class
GroupAttachedLeafClass
,
unsigned
NbSymbAttributes
,
unsigned
NbAttributesPerParticle
,
class
AttributeClass
=
FReal
>
class
FGroupTreeBerenger
{
public:
typedef
GroupAttachedLeafClass
BasicAttachedClass
;
typedef
FGroupOfParticles
<
FReal
,
NbSymbAttributes
,
NbAttributesPerParticle
,
AttributeClass
>
ParticleGroupClass
;
typedef
FGroupOfCells
<
CompositeCellClass
,
SymboleCellClass
,
PoleCellClass
,
LocalCellClass
>
CellGroupClass
;
protected:
//< height of the tree (1 => only the root)
const
int
treeHeight
;
//< max number of cells in a block
const
int
nbElementsPerBlock
;
//< all the blocks of the tree
std
::
vector
<
CellGroupClass
*>*
cellBlocksPerLevel
;
//< all the blocks of leaves
std
::
vector
<
ParticleGroupClass
*>
particleBlocks
;
//< the space system center
const
FPoint
<
FReal
>
boxCenter
;
//< the space system corner (used to compute morton index)
const
FPoint
<
FReal
>
boxCorner
;
//< the space system width
const
FReal
boxWidth
;
//< the width of a box at width level
const
FReal
boxWidthAtLeafLevel
;
public:
typedef
typename
std
::
vector
<
CellGroupClass
*>::
iterator
CellGroupIterator
;
typedef
typename
std
::
vector
<
CellGroupClass
*>::
const_iterator
CellGroupConstIterator
;
typedef
typename
std
::
vector
<
ParticleGroupClass
*>::
iterator
ParticleGroupIterator
;
typedef
typename
std
::
vector
<
ParticleGroupClass
*>::
const_iterator
ParticleGroupConstIterator
;
/**
* This constructor create a group tree from a particle container index.
* The morton index are computed and the particles are sorted in a first stage.
* Then the leaf level is done.
* Finally the other leve are proceed one after the other.
* It should be easy to make it parallel using for and tasks.
* If no limite give inLeftLimite = -1
*/
template
<
class
ParticleContainer
>
FGroupTreeBerenger
(
const
int
inTreeHeight
,
const
FReal
inBoxWidth
,
const
FPoint
<
FReal
>&
inBoxCenter
,
const
int
inNbElementsPerBlock
,
ParticleContainer
*
inParticlesContainer
,
const
bool
particlesAreSorted
,
std
::
vector
<
MortonIndex
>
const
&
distributedMortonIndex
,
MortonIndex
inLeftLimite
=
-
1
)
:
treeHeight
(
inTreeHeight
),
nbElementsPerBlock
(
inNbElementsPerBlock
),
cellBlocksPerLevel
(
nullptr
),
boxCenter
(
inBoxCenter
),
boxCorner
(
inBoxCenter
,
-
(
inBoxWidth
/
2
)),
boxWidth
(
inBoxWidth
),
boxWidthAtLeafLevel
(
inBoxWidth
/
FReal
(
1
<<
(
inTreeHeight
-
1
))){