From 2927a204aa962289088f9e2dd35888955a585525 Mon Sep 17 00:00:00 2001
From: Florent Pruvost <florent.pruvost@inria.fr>
Date: Wed, 7 Feb 2024 17:22:02 +0100
Subject: [PATCH] Add user mpi communicator feature.
---
compute/pzgetrf.c | 2 +-
compute/zgetrf.c | 2 +-
control/control.c | 40 +++++++++++++++++--
control/descriptor_ipiv.c | 18 +++++++--
coreblas/compute/CMakeLists.txt | 3 ++
example/lapack_to_chameleon/CTestLists.cmake | 15 +++++--
example/lapack_to_chameleon/step6.c | 20 +++++++++-
example/lapack_to_chameleon/step6.h | 8 ++--
include/chameleon.h | 27 +++++++++----
include/chameleon/runtime.h | 26 ++++++------
include/chameleon/runtime_struct.h | 12 ++++++
include/chameleon/struct.h | 12 ++++++
runtime/openmp/control/runtime_descriptor.c | 3 +-
.../openmp/control/runtime_descriptor_ipiv.c | 12 ++++--
runtime/parsec/control/runtime_descriptor.c | 4 +-
.../parsec/control/runtime_descriptor_ipiv.c | 12 ++++--
runtime/quark/control/runtime_descriptor.c | 3 +-
.../quark/control/runtime_descriptor_ipiv.c | 12 ++++--
runtime/starpu/codelets/codelet_map.c | 3 ++
runtime/starpu/codelets/codelet_zgersum.c | 2 +-
runtime/starpu/codelets/codelet_zlacpy.c | 4 +-
runtime/starpu/control/runtime_async.c | 6 +--
runtime/starpu/control/runtime_control.c | 22 +++++-----
runtime/starpu/control/runtime_descriptor.c | 10 ++---
.../starpu/control/runtime_descriptor_ipiv.c | 27 +++++++------
runtime/starpu/control/runtime_tags.c | 3 +-
runtime/starpu/include/chameleon_starpu.h.in | 4 +-
27 files changed, 223 insertions(+), 89 deletions(-)
diff --git a/compute/pzgetrf.c b/compute/pzgetrf.c
index 108c4a201..7e77479eb 100644
--- a/compute/pzgetrf.c
+++ b/compute/pzgetrf.c
@@ -400,7 +400,7 @@ void chameleon_pzgetrf( struct chameleon_pzgetrf_s *ws,
if ( (ws->alg == ChamGetrfNoPivPerColumn) ||
(ws->alg == ChamGetrfNoPiv ) )
{
- RUNTIME_ipiv_init( IPIV );
+ RUNTIME_ipiv_init( sequence, IPIV );
}
RUNTIME_options_finalize( &options, chamctxt );
diff --git a/compute/zgetrf.c b/compute/zgetrf.c
index 0a3682b20..9b002f5a6 100644
--- a/compute/zgetrf.c
+++ b/compute/zgetrf.c
@@ -280,7 +280,7 @@ CHAMELEON_zgetrf( int M, int N, CHAMELEON_Complex64_t *A, int LDA, int *IPIV )
if ( ( ws->alg == ChamGetrfPPivPerColumn ) ||
( ws->alg == ChamGetrfPPiv ) )
{
- RUNTIME_ipiv_gather( &descIPIV, IPIV, 0 );
+ RUNTIME_ipiv_gather( sequence, &descIPIV, IPIV, 0 );
}
chameleon_sequence_wait( chamctxt, sequence );
diff --git a/control/control.c b/control/control.c
index c3cacb10a..e1775a36e 100644
--- a/control/control.c
+++ b/control/control.c
@@ -35,7 +35,8 @@
*
* @ingroup Control
*
- * @brief Initialize CHAMELEON.
+ * @brief Initialize CHAMELEON with number of cpus and gpus (using
+ * MPI_COMM_WORLD).
*
******************************************************************************
*
@@ -59,7 +60,8 @@ int __chameleon_init(int cores, int gpus)
*
* @ingroup Control
*
- * @brief Initialize CHAMELEON.
+ * @brief Initialize CHAMELEON with number of cpus and gpus and threads per
+ * worker (using MPI_COMM_WORLD).
*
******************************************************************************
*
@@ -78,6 +80,37 @@ int __chameleon_init(int cores, int gpus)
*
*/
int __chameleon_initpar(int ncpus, int ngpus, int nthreads_per_worker)
+{
+ return __chameleon_initparcomm( ncpus, ngpus, nthreads_per_worker, MPI_COMM_WORLD );
+}
+
+/**
+ *
+ * @ingroup Control
+ *
+ * @brief Initialize CHAMELEON with number of cpus and gpus and threads per
+ * worker and using a given MPI communicator.
+ *
+ ******************************************************************************
+ *
+ * @param[in] ncpus
+ * Number of cores to use.
+ *
+ * @param[in] ngpus
+ * Number of cuda devices to use.
+ *
+ * @param[in] nthreads_per_worker
+ * Number of threads per worker (cpu, cuda device).
+ *
+ * @param[in] comm
+ * The MPI communicator.
+ *
+ ******************************************************************************
+ *
+ * @retval CHAMELEON_SUCCESS successful exit
+ *
+ */
+int __chameleon_initparcomm(int ncpus, int ngpus, int nthreads_per_worker, MPI_Comm comm)
{
CHAM_context_t *chamctxt;
@@ -124,6 +157,7 @@ int __chameleon_initpar(int ncpus, int ngpus, int nthreads_per_worker)
#endif
chamctxt->ncudas = ngpus;
+ chamctxt->comm = comm;
return RUNTIME_init( chamctxt, ncpus, ngpus, nthreads_per_worker );
}
@@ -145,7 +179,7 @@ int __chameleon_finalize(void)
chameleon_error("CHAMELEON_Finalize", "CHAMELEON not initialized");
return CHAMELEON_ERR_NOT_INITIALIZED;
}
- RUNTIME_flush();
+ RUNTIME_flush(chamctxt);
# if !defined(CHAMELEON_SIMULATION)
RUNTIME_barrier(chamctxt);
# endif
diff --git a/control/descriptor_ipiv.c b/control/descriptor_ipiv.c
index 784a0ef7b..4f697b740 100644
--- a/control/descriptor_ipiv.c
+++ b/control/descriptor_ipiv.c
@@ -210,7 +210,7 @@ int CHAMELEON_Ipiv_Destroy(CHAM_ipiv_t **ipivptr)
int CHAMELEON_Ipiv_Flush( const CHAM_ipiv_t *ipiv,
const RUNTIME_sequence_t *sequence )
{
- RUNTIME_ipiv_flush( ipiv, sequence );
+ RUNTIME_ipiv_flush( sequence, ipiv );
return CHAMELEON_SUCCESS;
}
@@ -240,6 +240,18 @@ int CHAMELEON_Ipiv_Flush( const CHAM_ipiv_t *ipiv,
*/
int CHAMELEON_Ipiv_Gather( CHAM_ipiv_t *ipivdesc, int *ipiv, int root )
{
- RUNTIME_ipiv_gather( ipivdesc, ipiv, root );
- return CHAMELEON_SUCCESS;
+ CHAM_context_t *chamctxt = chameleon_context_self();
+ RUNTIME_sequence_t *sequence = NULL;
+ int status;
+
+ chameleon_sequence_create( chamctxt, &sequence );
+
+ /* Submit the tasks to collect the ipiv array on root */
+ RUNTIME_ipiv_gather( sequence, ipivdesc, ipiv, root );
+
+ /* Wait for the end */
+ chameleon_sequence_wait( chamctxt, sequence );
+ status = sequence->status;
+ chameleon_sequence_destroy( chamctxt, sequence );
+ return status;
}
diff --git a/coreblas/compute/CMakeLists.txt b/coreblas/compute/CMakeLists.txt
index 137adfbba..bd4ad9f21 100644
--- a/coreblas/compute/CMakeLists.txt
+++ b/coreblas/compute/CMakeLists.txt
@@ -164,6 +164,9 @@ endif()
target_link_libraries(coreblas PRIVATE MORSE::LAPACKE)
target_link_libraries(coreblas PRIVATE MORSE::CBLAS)
target_link_libraries(coreblas PUBLIC MORSE::M)
+if (CHAMELEON_USE_MPI)
+ target_link_libraries(coreblas PUBLIC MPI::MPI_C)
+endif()
# export target coreblas
install(EXPORT coreblasTargets
diff --git a/example/lapack_to_chameleon/CTestLists.cmake b/example/lapack_to_chameleon/CTestLists.cmake
index 774e12e2c..2d3e86133 100644
--- a/example/lapack_to_chameleon/CTestLists.cmake
+++ b/example/lapack_to_chameleon/CTestLists.cmake
@@ -9,10 +9,19 @@ set(TESTLIST
step3
step4
step5
- step6
- step7
)
foreach(test ${TESTLIST})
- add_test(example_ltm_${test} ./${prec}${test})
+ add_test(example_ltm_${test} ./${test})
endforeach()
+
+if (CHAMELEON_USE_MPI AND MPI_C_FOUND)
+ set(MPITESTLIST
+ step6
+ step7
+ )
+ set ( MPICMD mpiexec --bind-to none -n 2 )
+ foreach(test ${MPITESTLIST})
+ add_test(example_ltm_${test} ${MPICMD} ./${test})
+ endforeach()
+endif()
diff --git a/example/lapack_to_chameleon/step6.c b/example/lapack_to_chameleon/step6.c
index 76a2b4fa0..181dfda1d 100644
--- a/example/lapack_to_chameleon/step6.c
+++ b/example/lapack_to_chameleon/step6.c
@@ -84,8 +84,19 @@ int main(int argc, char *argv[]) {
NCPU = iparam[IPARAM_THRDNBR];
NGPU = iparam[IPARAM_NCUDAS];
- /* Initialize CHAMELEON with main parameters */
- int rc = CHAMELEON_Init( NCPU, NGPU );
+ /* Example of Chameleon init with a given MPI communicator */
+#if defined(CHAMELEON_USE_MPI)
+ MPI_Init(NULL, NULL);
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ MPI_Comm comm;
+ MPI_Comm_split(MPI_COMM_WORLD, rank, rank, &comm);
+#else
+ MPI_Comm comm = 0;
+#endif
+
+ /* Initialize CHAMELEON with main parameters */
+ int rc = CHAMELEON_InitParComm( NCPU, NGPU, 1, comm );
if (rc != CHAMELEON_SUCCESS) {
goto finalize;
}
@@ -235,5 +246,10 @@ finalize:
/* Finalize CHAMELEON */
CHAMELEON_Finalize();
+#if defined(CHAMELEON_USE_MPI)
+ MPI_Comm_free(&comm);
+ MPI_Finalize();
+#endif
+
return EXIT_SUCCESS;
}
diff --git a/example/lapack_to_chameleon/step6.h b/example/lapack_to_chameleon/step6.h
index b834863c1..ecc33fd72 100644
--- a/example/lapack_to_chameleon/step6.h
+++ b/example/lapack_to_chameleon/step6.h
@@ -26,15 +26,15 @@
/* Specific includes for step 6 */
#include <coreblas/lapacke.h>
#include <chameleon.h>
-// #if defined(CHAMELEON_USE_MPI)
-// #include <mpi.h>
-// #endif
+#if defined(CHAMELEON_USE_MPI)
+#include <mpi.h>
+#endif
/* Integer parameters for step6 */
enum iparam_step6 {
IPARAM_THRDNBR, /* Number of cores */
IPARAM_NCUDAS, /* Number of cuda devices */
- IPARAM_NMPI, /* Number of cuda devices */
+ IPARAM_NMPI, /* Number of MPI PROCS */
IPARAM_N, /* Number of columns of the matrix */
IPARAM_NB, /* Number of columns in a tile */
IPARAM_IB, /* Inner-blocking size */
diff --git a/include/chameleon.h b/include/chameleon.h
index 2abadc0df..649103772 100644
--- a/include/chameleon.h
+++ b/include/chameleon.h
@@ -117,6 +117,7 @@ int CHAMELEON_Initialized (void);
int CHAMELEON_My_Mpi_Rank (void) __attribute__((deprecated));
int __chameleon_init (int nworkers, int ncudas);
int __chameleon_initpar (int nworkers, int ncudas, int nthreads_per_worker);
+int __chameleon_initparcomm (int nworkers, int ncudas, int nthreads_per_worker, MPI_Comm comm);
int __chameleon_finalize (void);
int CHAMELEON_Pause (void);
int CHAMELEON_Resume (void);
@@ -237,16 +238,23 @@ void CHAMELEON_Ipiv_Print ( const CHAM_ipiv_t *ipiv );
*
*/
#if defined(CHAMELEON_SCHED_OPENMP)
-#define CHAMELEON_Init( _nworkers_, _ncudas_ ) \
+
+#define CHAMELEON_Init( _nworkers_, _ncudas_ ) \
__chameleon_init( (_nworkers_), (_ncudas_) ); \
- _Pragma("omp parallel") \
- _Pragma("omp master") \
+ _Pragma("omp parallel") \
+ _Pragma("omp master") \
{
-#define CHAMELEON_InitPar( _nworkers_, _ncudas_, _nthreads_per_worker_ ) \
+#define CHAMELEON_InitPar( _nworkers_, _ncudas_, _nthreads_per_worker_ ) \
__chameleon_initpar( (_nworkers_), (_ncudas_), (_nthreads_per_worker_) ); \
- _Pragma("omp parallel")\
- _Pragma("omp master")\
+ _Pragma("omp parallel") \
+ _Pragma("omp master") \
+ {
+
+#define CHAMELEON_InitParComm( _nworkers_, _ncudas_, _nthreads_per_worker_, _comm_ ) \
+ __chameleon_initparcomm( (_nworkers_), (_ncudas_), (_nthreads_per_worker_), (_comm_) ); \
+ _Pragma("omp parallel") \
+ _Pragma("omp master") \
{
#define CHAMELEON_Finalize() \
@@ -255,11 +263,14 @@ void CHAMELEON_Ipiv_Print ( const CHAM_ipiv_t *ipiv );
#else
-#define CHAMELEON_Init( _nworkers_, _ncudas_ ) \
+#define CHAMELEON_Init( _nworkers_, _ncudas_ ) \
__chameleon_init( (_nworkers_), (_ncudas_) );
#define CHAMELEON_InitPar( _nworkers_, _ncudas_, _nthreads_per_worker_ ) \
- __chameleon_initpar( (_nworkers_), (_ncudas_), (_nthreads_per_worker_) );
+ __chameleon_initpar( (_nworkers_), (_ncudas_), (_nthreads_per_worker_), MPI_COMM_WORLD );
+
+#define CHAMELEON_InitParComm( _nworkers_, _ncudas_, _nthreads_per_worker_, _comm_ ) \
+ __chameleon_initparcomm( (_nworkers_), (_ncudas_), (_nthreads_per_worker_), (_comm_) );
#define CHAMELEON_Finalize() \
__chameleon_finalize();
diff --git a/include/chameleon/runtime.h b/include/chameleon/runtime.h
index dfe7bb3d9..12ec0a263 100644
--- a/include/chameleon/runtime.h
+++ b/include/chameleon/runtime.h
@@ -480,7 +480,7 @@ RUNTIME_desc_flush( const CHAM_desc_t *desc,
* This function flushes all data from the distributed cache of the runtime system.
*/
void
-RUNTIME_flush( );
+RUNTIME_flush( CHAM_context_t *chamctxt );
/**
* @brief Flush a single piece of data.
@@ -707,8 +707,19 @@ void RUNTIME_sdisplay_oneprofile (cham_tasktype_t task);
void RUNTIME_ipiv_create ( CHAM_ipiv_t *ipiv );
void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv );
-void RUNTIME_ipiv_init ( CHAM_ipiv_t *ipiv );
-void RUNTIME_ipiv_gather ( CHAM_ipiv_t *desc, int *ipiv, int node );
+void RUNTIME_ipiv_init ( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *ipiv );
+void RUNTIME_ipiv_gather ( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *desc, int *ipiv, int node );
+
+void RUNTIME_ipiv_flushk ( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv, int m );
+void RUNTIME_ipiv_flush ( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv );
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+ CHAM_ipiv_t *ws, int k, int h );
+void RUNTIME_perm_flushk( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv, int m );
void *RUNTIME_ipiv_getaddr ( const CHAM_ipiv_t *ipiv, int m );
void *RUNTIME_nextpiv_getaddr( const CHAM_ipiv_t *ipiv, int m, int h );
@@ -726,15 +737,6 @@ RUNTIME_pivot_getaddr( CHAM_ipiv_t *ipiv, int m, int h ) {
}
}
-void RUNTIME_ipiv_flushk ( const RUNTIME_sequence_t *sequence,
- const CHAM_ipiv_t *ipiv, int m );
-void RUNTIME_ipiv_flush ( const CHAM_ipiv_t *ipiv,
- const RUNTIME_sequence_t *sequence );
-void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
- CHAM_ipiv_t *ws, int k, int h );
-void RUNTIME_perm_flushk( const RUNTIME_sequence_t *sequence,
- const CHAM_ipiv_t *ipiv, int m );
-
/**
* @}
*/
diff --git a/include/chameleon/runtime_struct.h b/include/chameleon/runtime_struct.h
index 3028d328b..e0282d6ae 100644
--- a/include/chameleon/runtime_struct.h
+++ b/include/chameleon/runtime_struct.h
@@ -23,6 +23,17 @@
#ifndef _chameleon_runtime_struct_h_
#define _chameleon_runtime_struct_h_
+#if defined(CHAMELEON_USE_MPI)
+#include <mpi.h>
+#else
+#ifndef MPI_Comm
+typedef uintptr_t MPI_Comm;
+#endif
+#ifndef MPI_COMM_WORLD
+#define MPI_COMM_WORLD 0
+#endif
+#endif
+
BEGIN_C_DECLS
/**
@@ -70,6 +81,7 @@ typedef struct runtime_sequence_s {
int status; /**< Return status registered by the tasks for the request */
RUNTIME_request_t *request; /**< Pointer to the request that failed if any, NULL otherwise */
void *schedopt; /**< Specific runtime data pointer to handle the sequence */
+ MPI_Comm comm; /**< MPI communicator */
} RUNTIME_sequence_t;
/**
diff --git a/include/chameleon/struct.h b/include/chameleon/struct.h
index 00a79664c..539049e3c 100644
--- a/include/chameleon/struct.h
+++ b/include/chameleon/struct.h
@@ -30,6 +30,17 @@
#include "chameleon/constants.h"
#include "chameleon/runtime_struct.h"
+#if defined(CHAMELEON_USE_MPI)
+#include <mpi.h>
+#else
+#ifndef MPI_Comm
+typedef uintptr_t MPI_Comm;
+#endif
+#ifndef MPI_COMM_WORLD
+#define MPI_COMM_WORLD 0
+#endif
+#endif
+
BEGIN_C_DECLS
#define CHAMELEON_TILE_FULLRANK (1 << 0)
@@ -191,6 +202,7 @@ typedef struct chameleon_context_s {
int lookahead; // depth of the look ahead in algorithms
void *schedopt; // structure for runtimes
int mpi_outer_init; // MPI has been initialized outside our functions
+ MPI_Comm comm; // MPI communicator
} CHAM_context_t;
static inline void *
diff --git a/runtime/openmp/control/runtime_descriptor.c b/runtime/openmp/control/runtime_descriptor.c
index 38ea0a3b1..9ba825542 100644
--- a/runtime/openmp/control/runtime_descriptor.c
+++ b/runtime/openmp/control/runtime_descriptor.c
@@ -69,8 +69,9 @@ RUNTIME_desc_flush( const CHAM_desc_t *desc,
void
-RUNTIME_flush( )
+RUNTIME_flush( CHAM_context_t *chamctxt )
{
+ (void)chamctxt;
return;
}
diff --git a/runtime/openmp/control/runtime_descriptor_ipiv.c b/runtime/openmp/control/runtime_descriptor_ipiv.c
index 0743412c4..3413eca76 100644
--- a/runtime/openmp/control/runtime_descriptor_ipiv.c
+++ b/runtime/openmp/control/runtime_descriptor_ipiv.c
@@ -80,8 +80,8 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
(void)m;
}
-void RUNTIME_ipiv_flush( const CHAM_ipiv_t *ipiv,
- const RUNTIME_sequence_t *sequence )
+void RUNTIME_ipiv_flush( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv )
{
assert( 0 );
(void)ipiv;
@@ -107,15 +107,19 @@ void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
(void)h;
}
-void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+void RUNTIME_ipiv_init( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *ipiv )
{
assert( 0 );
+ (void)sequence;
(void)ipiv;
}
-void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+void RUNTIME_ipiv_gather( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *desc, int *ipiv, int node )
{
assert( 0 );
+ (void)sequence;
(void)desc;
(void)ipiv;
(void)node;
diff --git a/runtime/parsec/control/runtime_descriptor.c b/runtime/parsec/control/runtime_descriptor.c
index 360d67326..a52a37f08 100644
--- a/runtime/parsec/control/runtime_descriptor.c
+++ b/runtime/parsec/control/runtime_descriptor.c
@@ -345,8 +345,10 @@ int RUNTIME_desc_release( const CHAM_desc_t *desc )
/**
* Flush cached data
*/
-void RUNTIME_flush()
+void RUNTIME_flush( CHAM_context_t *chamctxt )
{
+ (void)chamctxt;
+ return;
}
void RUNTIME_desc_flush( const CHAM_desc_t *desc,
diff --git a/runtime/parsec/control/runtime_descriptor_ipiv.c b/runtime/parsec/control/runtime_descriptor_ipiv.c
index 970ad741c..f18cde2ea 100644
--- a/runtime/parsec/control/runtime_descriptor_ipiv.c
+++ b/runtime/parsec/control/runtime_descriptor_ipiv.c
@@ -80,8 +80,8 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
(void)m;
}
-void RUNTIME_ipiv_flush( const CHAM_ipiv_t *ipiv,
- const RUNTIME_sequence_t *sequence )
+void RUNTIME_ipiv_flush( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv )
{
assert( 0 );
(void)ipiv;
@@ -107,15 +107,19 @@ void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
(void)h;
}
-void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+void RUNTIME_ipiv_init( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *ipiv )
{
assert( 0 );
+ (void)sequence;
(void)ipiv;
}
-void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+void RUNTIME_ipiv_gather( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *desc, int *ipiv, int node )
{
assert( 0 );
+ (void)sequence;
(void)desc;
(void)ipiv;
(void)node;
diff --git a/runtime/quark/control/runtime_descriptor.c b/runtime/quark/control/runtime_descriptor.c
index 6301b9c14..2676e02e8 100644
--- a/runtime/quark/control/runtime_descriptor.c
+++ b/runtime/quark/control/runtime_descriptor.c
@@ -70,8 +70,9 @@ RUNTIME_desc_flush( const CHAM_desc_t *desc,
void
-RUNTIME_flush( )
+RUNTIME_flush( CHAM_context_t *chamctxt )
{
+ (void)chamctxt;
return;
}
diff --git a/runtime/quark/control/runtime_descriptor_ipiv.c b/runtime/quark/control/runtime_descriptor_ipiv.c
index f3ea5a113..02ea867a6 100644
--- a/runtime/quark/control/runtime_descriptor_ipiv.c
+++ b/runtime/quark/control/runtime_descriptor_ipiv.c
@@ -80,8 +80,8 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
(void)m;
}
-void RUNTIME_ipiv_flush( const CHAM_ipiv_t *ipiv,
- const RUNTIME_sequence_t *sequence )
+void RUNTIME_ipiv_flush( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv )
{
assert( 0 );
(void)ipiv;
@@ -107,15 +107,19 @@ void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
(void)h;
}
-void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+void RUNTIME_ipiv_init( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *ipiv )
{
assert( 0 );
+ (void)sequence;
(void)ipiv;
}
-void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+void RUNTIME_ipiv_gather( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *desc, int *ipiv, int node )
{
assert( 0 );
+ (void)sequence;
(void)desc;
(void)ipiv;
(void)node;
diff --git a/runtime/starpu/codelets/codelet_map.c b/runtime/starpu/codelets/codelet_map.c
index 2ef297b0d..50a93a67b 100644
--- a/runtime/starpu/codelets/codelet_map.c
+++ b/runtime/starpu/codelets/codelet_map.c
@@ -280,6 +280,7 @@ void INSERT_TASK_map( const RUNTIME_option_t *options,
case 1:
callback = options->profiling ? cl_map_one_callback : NULL;
rt_starpu_insert_task(
+ options->sequence->comm,
&cl_map_one,
/* Task codelet arguments */
STARPU_CL_ARGS, clargs, clargs_size,
@@ -300,6 +301,7 @@ void INSERT_TASK_map( const RUNTIME_option_t *options,
case 2:
callback = options->profiling ? cl_map_two_callback : NULL;
rt_starpu_insert_task(
+ options->sequence->comm,
&cl_map_two,
/* Task codelet arguments */
STARPU_CL_ARGS, clargs, clargs_size,
@@ -321,6 +323,7 @@ void INSERT_TASK_map( const RUNTIME_option_t *options,
case 3:
callback = options->profiling ? cl_map_three_callback : NULL;
rt_starpu_insert_task(
+ options->sequence->comm,
&cl_map_three,
/* Task codelet arguments */
STARPU_CL_ARGS, clargs, clargs_size,
diff --git a/runtime/starpu/codelets/codelet_zgersum.c b/runtime/starpu/codelets/codelet_zgersum.c
index 8f8b2eaeb..525674b23 100644
--- a/runtime/starpu/codelets/codelet_zgersum.c
+++ b/runtime/starpu/codelets/codelet_zgersum.c
@@ -128,7 +128,7 @@ RUNTIME_zgersum_submit_tree( const RUNTIME_option_t *options,
const CHAM_desc_t *A, int Am, int An )
{
#if defined(HAVE_STARPU_MPI_REDUX) && defined(CHAMELEON_USE_MPI)
- starpu_mpi_redux_data_prio_tree( MPI_COMM_WORLD,
+ starpu_mpi_redux_data_prio_tree( options->sequence->comm,
RTBLKADDR(A, ChamComplexDouble, Am, An),
options->priority + 1,
2 /* Binary tree */ );
diff --git a/runtime/starpu/codelets/codelet_zlacpy.c b/runtime/starpu/codelets/codelet_zlacpy.c
index af940bf28..0d998b617 100644
--- a/runtime/starpu/codelets/codelet_zlacpy.c
+++ b/runtime/starpu/codelets/codelet_zlacpy.c
@@ -120,9 +120,9 @@ insert_task_zlacpy_on_remote_node( const RUNTIME_option_t *options,
{
void (*callback)(void*) = options->profiling ? cl_zlacpy_callback : NULL;
#if defined(CHAMELEON_RUNTIME_SYNC)
- starpu_mpi_data_cpy_priority( handleB, handleA, MPI_COMM_WORLD, 0, callback, NULL, options->priority );
+ starpu_mpi_data_cpy_priority( handleB, handleA, options->sequence->comm, 0, callback, NULL, options->priority );
#else
- starpu_mpi_data_cpy_priority( handleB, handleA, MPI_COMM_WORLD, 1, callback, NULL, options->priority );
+ starpu_mpi_data_cpy_priority( handleB, handleA, options->sequence->comm, 1, callback, NULL, options->priority );
#endif
}
#endif
diff --git a/runtime/starpu/control/runtime_async.c b/runtime/starpu/control/runtime_async.c
index a439e5d8f..8804b1779 100644
--- a/runtime/starpu/control/runtime_async.c
+++ b/runtime/starpu/control/runtime_async.c
@@ -28,7 +28,7 @@ int RUNTIME_sequence_create( CHAM_context_t *chamctxt,
RUNTIME_sequence_t *sequence )
{
(void)chamctxt;
- (void)sequence;
+ sequence->comm = chamctxt->comm;
return CHAMELEON_SUCCESS;
}
@@ -58,10 +58,10 @@ int RUNTIME_sequence_wait( CHAM_context_t *chamctxt,
#if defined(CHAMELEON_USE_MPI)
# if defined(HAVE_STARPU_MPI_WAIT_FOR_ALL)
- starpu_mpi_wait_for_all(MPI_COMM_WORLD);
+ starpu_mpi_wait_for_all(sequence->comm);
# else
starpu_task_wait_for_all();
- starpu_mpi_barrier(MPI_COMM_WORLD);
+ starpu_mpi_barrier(sequence->comm);
# endif
#else
starpu_task_wait_for_all();
diff --git a/runtime/starpu/control/runtime_control.c b/runtime/starpu/control/runtime_control.c
index b8fd4003f..94031382b 100644
--- a/runtime/starpu/control/runtime_control.c
+++ b/runtime/starpu/control/runtime_control.c
@@ -100,7 +100,7 @@ void chameleon_starpu_parallel_worker_fini( starpu_sched_opt_t *sched_opt )
/**
*
*/
-static int chameleon_starpu_init( struct starpu_conf *conf )
+static int chameleon_starpu_init( MPI_Comm comm, struct starpu_conf *conf )
{
int hres = CHAMELEON_SUCCESS;
int rc;
@@ -118,7 +118,7 @@ static int chameleon_starpu_init( struct starpu_conf *conf )
# endif
# if defined(HAVE_STARPU_MPI_INIT_CONF)
- rc = starpu_mpi_init_conf(NULL, NULL, !flag, MPI_COMM_WORLD, conf);
+ rc = starpu_mpi_init_conf(NULL, NULL, !flag, comm, conf);
# else
rc = starpu_init(conf);
if (rc < 0) {
@@ -186,7 +186,7 @@ int RUNTIME_init( CHAM_context_t *chamctxt,
if ((ncpus == -1)||(nthreads_per_worker == -1))
{
- hres = chameleon_starpu_init( conf );
+ hres = chameleon_starpu_init( chamctxt->comm, conf );
chamctxt->nworkers = ncpus;
chamctxt->nthreads_per_worker = nthreads_per_worker;
@@ -202,7 +202,7 @@ int RUNTIME_init( CHAM_context_t *chamctxt,
conf->use_explicit_workers_bindid = 1;
- hres = chameleon_starpu_init( conf );
+ hres = chameleon_starpu_init( chamctxt->comm, conf );
chamctxt->nworkers = ncpus;
chamctxt->nthreads_per_worker = nthreads_per_worker;
@@ -300,11 +300,11 @@ void RUNTIME_barrier( CHAM_context_t *chamctxt )
#if defined(CHAMELEON_USE_MPI)
# if defined(HAVE_STARPU_MPI_WAIT_FOR_ALL)
- starpu_mpi_wait_for_all(MPI_COMM_WORLD);
- starpu_mpi_barrier(MPI_COMM_WORLD);
+ starpu_mpi_wait_for_all( chamctxt->comm );
+ starpu_mpi_barrier( chamctxt->comm );
# else
starpu_task_wait_for_all();
- starpu_mpi_barrier(MPI_COMM_WORLD);
+ starpu_mpi_barrier( chamctxt->comm );
# endif
#else
starpu_task_wait_for_all();
@@ -380,9 +380,9 @@ int RUNTIME_comm_rank( CHAM_context_t *chamctxt )
#if defined(CHAMELEON_USE_MPI)
# if defined(HAVE_STARPU_MPI_COMM_RANK)
- starpu_mpi_comm_rank( MPI_COMM_WORLD, &rank );
+ starpu_mpi_comm_rank( chamctxt->comm, &rank );
# else
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+ MPI_Comm_rank( chamctxt->comm, &rank );
# endif
#endif
@@ -398,9 +398,9 @@ int RUNTIME_comm_size( CHAM_context_t *chamctxt )
int size;
#if defined(CHAMELEON_USE_MPI)
# if defined(HAVE_STARPU_MPI_COMM_RANK)
- starpu_mpi_comm_size( MPI_COMM_WORLD, &size );
+ starpu_mpi_comm_size( chamctxt->comm, &size );
# else
- MPI_Comm_size( MPI_COMM_WORLD, &size );
+ MPI_Comm_size( chamctxt->comm, &size );
# endif
#else
size = 1;
diff --git a/runtime/starpu/control/runtime_descriptor.c b/runtime/starpu/control/runtime_descriptor.c
index 1e9866011..1c21c9e60 100644
--- a/runtime/starpu/control/runtime_descriptor.c
+++ b/runtime/starpu/control/runtime_descriptor.c
@@ -149,7 +149,7 @@ void RUNTIME_desc_create( CHAM_desc_t *desc )
* Book the number of tags required to describe this matrix
*/
{
- chameleon_starpu_tag_init();
+ chameleon_starpu_tag_init( );
desc->mpitag = chameleon_starpu_tag_book( nbtiles );
if ( desc->mpitag == -1 ) {
@@ -267,10 +267,10 @@ int RUNTIME_desc_release( const CHAM_desc_t *desc )
/**
* Flush cached data
*/
-void RUNTIME_flush()
+void RUNTIME_flush( CHAM_context_t *chamctxt )
{
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush_all_data(MPI_COMM_WORLD);
+ starpu_mpi_cache_flush_all_data(chamctxt->comm);
#endif
}
@@ -317,7 +317,7 @@ void RUNTIME_data_flush( const RUNTIME_sequence_t *sequence,
}
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush( MPI_COMM_WORLD, *handlebis );
+ starpu_mpi_cache_flush( sequence->comm, *handlebis );
#endif
if ( local ) {
@@ -345,7 +345,7 @@ void RUNTIME_data_migrate( const RUNTIME_sequence_t *sequence,
old_rank = starpu_mpi_data_get_rank( lhandle );
if ( old_rank != new_rank ) {
- starpu_mpi_data_migrate( MPI_COMM_WORLD, lhandle, new_rank );
+ starpu_mpi_data_migrate( sequence->comm, lhandle, new_rank );
}
(void)sequence;
diff --git a/runtime/starpu/control/runtime_descriptor_ipiv.c b/runtime/starpu/control/runtime_descriptor_ipiv.c
index d3a3d82c5..3e569e245 100644
--- a/runtime/starpu/control/runtime_descriptor_ipiv.c
+++ b/runtime/starpu/control/runtime_descriptor_ipiv.c
@@ -39,7 +39,7 @@ void RUNTIME_ipiv_create( CHAM_ipiv_t *ipiv )
* One per handle type
*/
{
- chameleon_starpu_tag_init();
+ chameleon_starpu_tag_init( );
ipiv->mpitag_ipiv = chameleon_starpu_tag_book( (int64_t)(ipiv->mt) * 5 );
if ( ipiv->mpitag_ipiv == -1 ) {
chameleon_fatal_error("RUNTIME_ipiv_create", "Can't pursue computation since no more tags are available for ipiv structure");
@@ -223,7 +223,7 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
if ( *handle != NULL ) {
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+ starpu_mpi_cache_flush( sequence->comm, *handle );
if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
#endif
{
@@ -236,7 +236,7 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
if ( *handle != NULL ) {
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+ starpu_mpi_cache_flush( sequence->comm, *handle );
if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
#endif
{
@@ -249,8 +249,8 @@ void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
(void)m;
}
-void RUNTIME_ipiv_flush( const CHAM_ipiv_t *ipiv,
- const RUNTIME_sequence_t *sequence )
+void RUNTIME_ipiv_flush( const RUNTIME_sequence_t *sequence,
+ const CHAM_ipiv_t *ipiv )
{
int m;
@@ -272,7 +272,7 @@ void RUNTIME_perm_flushk( const RUNTIME_sequence_t *sequence,
if ( *handle != NULL ) {
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+ starpu_mpi_cache_flush( sequence->comm, *handle );
if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
#endif
{
@@ -285,7 +285,7 @@ void RUNTIME_perm_flushk( const RUNTIME_sequence_t *sequence,
if ( *handle != NULL ) {
#if defined(CHAMELEON_USE_MPI)
- starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+ starpu_mpi_cache_flush( sequence->comm, *handle );
if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
#endif
{
@@ -307,7 +307,7 @@ void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
#if defined(HAVE_STARPU_MPI_REDUX) && defined(CHAMELEON_USE_MPI)
#if !defined(HAVE_STARPU_MPI_REDUX_WRAPUP)
if ( h < ipiv->n ) {
- starpu_mpi_redux_data_prio_tree( MPI_COMM_WORLD, nextpiv,
+ starpu_mpi_redux_data_prio_tree( options->sequence->comm, nextpiv,
options->priority, 2 /* Binary tree */ );
}
#endif
@@ -343,7 +343,8 @@ struct starpu_codelet cl_ipiv_init = {
.nbuffers = 1,
};
-void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+void RUNTIME_ipiv_init( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *ipiv )
{
int64_t mt = ipiv->mt;
int64_t mb = ipiv->mb;
@@ -355,6 +356,7 @@ void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
int n = (m == (mt-1)) ? ipiv->m - m0 : mb;
rt_starpu_insert_task(
+ sequence->comm,
&cl_ipiv_init,
STARPU_VALUE, &m0, sizeof(int),
STARPU_VALUE, &n, sizeof(int),
@@ -363,7 +365,8 @@ void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
}
}
-void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+void RUNTIME_ipiv_gather( const RUNTIME_sequence_t *sequence,
+ CHAM_ipiv_t *desc, int *ipiv, int node )
{
int64_t mt = desc->mt;
int64_t mb = desc->mb;
@@ -387,7 +390,7 @@ void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
if (already_received == 0)
{
MPI_Status status;
- starpu_mpi_recv( ipiv_src, owner, tag, MPI_COMM_WORLD, &status );
+ starpu_mpi_recv( ipiv_src, owner, tag, sequence->comm, &status );
}
}
else if ( rank == owner )
@@ -396,7 +399,7 @@ void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
int already_sent = starpu_mpi_cached_send_set( ipiv_src, node );
if (already_sent == 0)
{
- starpu_mpi_send( ipiv_src, node, tag, MPI_COMM_WORLD );
+ starpu_mpi_send( ipiv_src, node, tag, sequence->comm );
}
}
}
diff --git a/runtime/starpu/control/runtime_tags.c b/runtime/starpu/control/runtime_tags.c
index 57c9f859e..e8d8c6628 100644
--- a/runtime/starpu/control/runtime_tags.c
+++ b/runtime/starpu/control/runtime_tags.c
@@ -58,7 +58,8 @@ chameleon_starpu_tag_init( void )
int ok = 0;
void *tag_ub_p = NULL;
- starpu_mpi_comm_get_attr( MPI_COMM_WORLD, STARPU_MPI_TAG_UB, &tag_ub_p, &ok );
+ CHAM_context_t *chamctxt = chameleon_context_self();
+ starpu_mpi_comm_get_attr( chamctxt->comm, STARPU_MPI_TAG_UB, &tag_ub_p, &ok );
starpu_tag_ub = (uint64_t)((intptr_t)tag_ub_p);
if ( !ok ) {
diff --git a/runtime/starpu/include/chameleon_starpu.h.in b/runtime/starpu/include/chameleon_starpu.h.in
index fd6d0e468..7aa8e9ba6 100644
--- a/runtime/starpu/include/chameleon_starpu.h.in
+++ b/runtime/starpu/include/chameleon_starpu.h.in
@@ -131,10 +131,10 @@ void *RUNTIME_data_getaddr_withconversion( const RUNTIME_option_t *options,
#if defined(CHAMELEON_RUNTIME_SYNC)
#define rt_starpu_insert_task( _codelet_, ... ) \
- starpu_mpi_insert_task( MPI_COMM_WORLD, (_codelet_), STARPU_TASK_SYNCHRONOUS, 1, ##__VA_ARGS__ )
+ starpu_mpi_insert_task( options->sequence->comm, (_codelet_), STARPU_TASK_SYNCHRONOUS, 1, ##__VA_ARGS__ )
#else
#define rt_starpu_insert_task( _codelet_, ... ) \
- starpu_mpi_insert_task( MPI_COMM_WORLD, (_codelet_), ##__VA_ARGS__ )
+ starpu_mpi_insert_task( options->sequence->comm, (_codelet_), ##__VA_ARGS__ )
#endif
#else
--
GitLab