Introduce MORSE_Distributed_start, MORSE_Distributed_stop,

MORSE_Distributed_size, MORSE_Distributed_rank so that applications do not
hardcode the use of MPI.

Introduce RUNTIME_distributed_rank, RUNTIME_distributed_size,
RUNTIME_distributed_barrier, so that MORSE does not hardcode the use of MPI
either.

This allows to use simgrid-mpi.

CMakeLists.txt

@@ -625,6 +625,17 @@ if (MORSE_DISTRIB_DIR OR EXISTS "${CMAKE_SOURCE_DIR}/cmake_modules/")
                     " - starpu_mpi_data_register() test fails in StarPU${ColourReset}")
                     message("-- ${Red}Check in CMakeFiles/CMakeError.log to figure out why it fails${ColourReset}")
                 endif()
+		unset(STARPU_MPI_COMM_RANK_FOUND CACHE)
+		check_function_exists(starpu_mpi_comm_rank STARPU_MPI_COMM_RANK_FOUND)
+		if ( STARPU_MPI_COMM_RANK_FOUND )
+                    message("-- ${Blue}Add definition HAVE_STARPU_MPI_COMM_RANK - Activate"
+                    "use of starpu_mpi_comm_rank() in Chameleon with StarPU${ColourReset}")
+                    add_definitions(-DHAVE_STARPU_MPI_COMM_RANK)
+                else()
+                    message("-- ${Red}Looking for starpu with starpu_mpi_data_register"
+                    " - starpu_mpi_comm_rank() test fails in StarPU${ColourReset}")
+                    message("-- ${Red}Check in CMakeFiles/CMakeError.log to figure out why it fails${ColourReset}")
+                endif()
             endif()
             if(HWLOC_FOUND AND HWLOC_LIBRARY_DIRS)
                 # the RPATH to be used when installing

control/control.c

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -138,8 +138,8 @@ int MORSE_Finalize(void)
     magma_finalize();
 #endif
     morse_context_destroy();
+    RUNTIME_distributed_barrier();
 #if defined(CHAMELEON_USE_MPI)
-    MPI_Barrier(MPI_COMM_WORLD);
     if (!morse->mpi_outer_init)
         MPI_Finalize();
 #endif
@@ -193,3 +193,75 @@ int MORSE_Resume(void)
     RUNTIME_resume(morse);
     return MORSE_SUCCESS;
 }
+
+/** ***************************************************************************
+ *
+ * @ingroup Control
+ *
+ *  MORSE_Distributed_start - Prepare the distributed processes for computation
+ *
+ ******************************************************************************
+ *
+ * @return
+ *          \retval MORSE_SUCCESS successful exit
+ *
+ *****************************************************************************/
+int MORSE_Distributed_start(void)
+{
+    RUNTIME_distributed_barrier ();
+    return MORSE_SUCCESS;
+}
+
+/** ***************************************************************************
+ *
+ * @ingroup Control
+ *
+ *  MORSE_Distributed_stop - Clean the distributed processes after computation
+ *
+ ******************************************************************************
+ *
+ * @return
+ *          \retval MORSE_SUCCESS successful exit
+ *
+ *****************************************************************************/
+int MORSE_Distributed_stop(void)
+{
+    RUNTIME_distributed_barrier ();
+    return MORSE_SUCCESS;
+}
+
+/** ***************************************************************************
+ *
+ * @ingroup Control
+ *
+ *  MORSE_Distributed_size - Return the size of the distributed computation
+ *
+ ******************************************************************************
+ *
+ * @return
+ *          \retval MORSE_SUCCESS successful exit
+ *
+ *****************************************************************************/
+int MORSE_Distributed_size( int *size )
+{
+    RUNTIME_distributed_size (size);
+    return MORSE_SUCCESS;
+}
+
+/** ***************************************************************************
+ *
+ * @ingroup Control
+ *
+ *  MORSE_Distributed_rank - Return the rank of the distributed computation
+ *
+ ******************************************************************************
+ *
+ * @return
+ *          \retval MORSE_SUCCESS successful exit
+ *
+ *****************************************************************************/
+int MORSE_Distributed_rank( int *rank )
+{
+    RUNTIME_distributed_rank (rank);
+    return MORSE_SUCCESS;
+}

control/descriptor.c

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -77,11 +77,7 @@ MORSE_desc_t morse_desc_init(MORSE_enum dtyp, int mb, int nb, int bsiz,
     desc.alloc_mat = 1;
     desc.register_mat = 1;
 
-#if defined(CHAMELEON_USE_MPI)
-    MPI_Comm_rank( MPI_COMM_WORLD, &(desc.myrank) );
-#else
-    desc.myrank = 0;
-#endif
+    RUNTIME_distributed_rank( &(desc.myrank) );
 
     // Grid size
     desc.p = p;
@@ -166,11 +162,7 @@ MORSE_desc_t morse_desc_init_diag(MORSE_enum dtyp, int mb, int nb, int bsiz,
     desc.alloc_mat = 1;
     desc.register_mat = 1;
 
-#if defined(CHAMELEON_USE_MPI)
-    MPI_Comm_rank( MPI_COMM_WORLD, &(desc.myrank) );
-#else
-    desc.myrank = 0;
-#endif
+    RUNTIME_distributed_rank( &(desc.myrank) );
 
     // Grid size
     desc.p = p;
@@ -258,11 +250,7 @@ MORSE_desc_t morse_desc_init_user(MORSE_enum dtyp, int mb, int nb, int bsiz,
     desc.alloc_mat = 1;
     desc.register_mat = 1;
 
-#if defined(CHAMELEON_USE_MPI)
-    MPI_Comm_rank( MPI_COMM_WORLD, &(desc.myrank) );
-#else
-    desc.myrank = 0;
-#endif
+    RUNTIME_distributed_rank( &(desc.myrank) );
 
     // Grid size
     desc.p = p;
@@ -418,7 +406,7 @@ int morse_desc_mat_free( MORSE_desc_t *desc )
     if (desc->mat != NULL  &&
         desc->use_mat == 1 &&
         desc->alloc_mat == 1) {
-#ifndef CHAMELEON_SIMULATION
+#if !defined(CHAMELEON_SIMULATION) || defined(CHAMELEON_USE_MPI)
         free(desc->mat);
 #endif
         desc->mat = NULL;
@@ -508,7 +496,7 @@ int MORSE_Desc_Create(MORSE_desc_t **desc, void *mat, MORSE_enum dtyp, int mb, i
 
     if (mat == NULL) {
 
-#ifdef CHAMELEON_SIMULATION
+#if defined(CHAMELEON_SIMULATION) && !defined(CHAMELEON_USE_MPI)
         (**desc).mat = (void*) 1;
 #else
         /* TODO: a call to morse_desc_mat_alloc should be made, but require to

include/morse.h.in

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -81,6 +81,10 @@ int MORSE_Pause          (void);
 int MORSE_Resume         (void);
 int MORSE_Lapack_to_Tile (void *Af77, int LDA, MORSE_desc_t *A);
 int MORSE_Tile_to_Lapack (MORSE_desc_t *A, void *Af77, int LDA);
+int MORSE_Distributed_start(void);
+int MORSE_Distributed_stop (void);
+int MORSE_Distributed_size (int*size);
+int MORSE_Distributed_rank (int*rank);
 
 /* Descriptor */
 int MORSE_Desc_Create  (MORSE_desc_t **desc, void *mat, MORSE_enum dtyp,

include/runtime.h

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -54,6 +54,9 @@ void  RUNTIME_finalize_scheduler (MORSE_context_t*);
 void  RUNTIME_barrier            (MORSE_context_t*);
 void  RUNTIME_pause              (MORSE_context_t*);
 void  RUNTIME_resume             (MORSE_context_t*);
+void  RUNTIME_distributed_rank   (int*);
+void  RUNTIME_distributed_size   (int*);
+void  RUNTIME_distributed_barrier(void);
 
 /*******************************************************************************
  * RUNTIME Descriptor

runtime/quark/control/runtime_control.c

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -86,3 +86,29 @@ void RUNTIME_resume( MORSE_context_t *morse )
     (void)morse;
     return;
 }
+
+/*******************************************************************************
+ *  This returns the rank of this process
+ **/
+void RUNTIME_distributed_rank( int *rank )
+{
+    *rank = 0;
+    return;
+}
+
+/*******************************************************************************
+ *  This returns the size of the distributed computation
+ **/
+void RUNTIME_distributed_size( int *size )
+{
+    *size = 1;
+    return;
+}
+
+/*******************************************************************************
+ *  Barrier between processes of the distributed computation
+ **/
+void RUNTIME_distributed_barrier( void )
+{
+    return;
+}

runtime/starpu/control/runtime_control.c

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -107,8 +107,8 @@ int RUNTIME_init_scheduler( MORSE_context_t *morse, int ncpus, int ncudas, int n
         int flag = 0;
         MPI_Initialized( &flag );
         starpu_mpi_init(NULL, NULL, !flag);
-        MPI_Comm_rank(MPI_COMM_WORLD, &(morse->my_mpi_rank));
-        MPI_Comm_size(MPI_COMM_WORLD, &(morse->mpi_comm_size));
+        starpu_mpi_comm_rank(MPI_COMM_WORLD, &(morse->my_mpi_rank));
+        starpu_mpi_comm_size(MPI_COMM_WORLD, &(morse->mpi_comm_size));
     }
 #endif
 
@@ -178,3 +178,48 @@ void RUNTIME_resume( MORSE_context_t *morse )
     starpu_resume();
     return;
 }
+
+/*******************************************************************************
+ *  This returns the rank of this process
+ **/
+void RUNTIME_distributed_rank( int *rank )
+{
+#if defined(CHAMELEON_USE_MPI)
+#  if defined(HAVE_STARPU_MPI_RANK)
+    starpu_mpi_comm_rank(MPI_COMM_WORLD, rank);
+#  else
+    MPI_Comm_rank(MPI_COMM_WORLD, rank);
+#  endif
+#else
+    *rank = 0;
+#endif
+    return;
+}
+
+/*******************************************************************************
+ *  This returns the size of the distributed computation
+ **/
+void RUNTIME_distributed_size( int *size )
+{
+#if defined(CHAMELEON_USE_MPI)
+#  if defined(HAVE_STARPU_MPI_RANK)
+    starpu_mpi_comm_size(MPI_COMM_WORLD, size);
+#  else
+    MPI_Comm_size(MPI_COMM_WORLD, size);
+#  endif
+#else
+    *size = 1;
+#endif
+    return;
+}
+
+/*******************************************************************************
+ *  Barrier between processes of the distributed computation
+ **/
+void RUNTIME_distributed_barrier( void )
+{
+#if defined(CHAMELEON_USE_MPI)
+    starpu_mpi_barrier(MPI_COMM_WORLD);
+#endif
+    return;
+}

timing/timing.c

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -622,7 +622,7 @@ main(int argc, char *argv[]) {
         MORSE_Enable(MORSE_ERRORS);
 
 #if defined(CHAMELEON_USE_MPI)
-    MPI_Comm_size( MPI_COMM_WORLD, &nbnode );
+    MORSE_Distributed_size( &nbnode );
     iparam[IPARAM_NMPI] = nbnode;
     /* Check P */
     if ( (iparam[IPARAM_P] > 1) &&

timing/timing.h

@@ -4,7 +4,7 @@
  *                          of Tennessee Research Foundation.
  *                          All rights reserved.
  * @copyright (c) 2012-2014 Inria. All rights reserved.
- * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+ * @copyright (c) 2012-2015 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
  *
  **/
 
@@ -185,8 +185,8 @@ enum dparam_timing {
  *
  */
 #if defined(CHAMELEON_USE_MPI)
-#define START_DISTRIBUTED()  MPI_Barrier(MPI_COMM_WORLD);
-#define STOP_DISTRIBUTED()   MPI_Barrier(MPI_COMM_WORLD);
+#define START_DISTRIBUTED()  MORSE_Distributed_start();
+#define STOP_DISTRIBUTED()   MORSE_Distributed_stop();
 #else
 #define START_DISTRIBUTED()  do {} while(0);
 #define STOP_DISTRIBUTED()   do {} while(0);