runtime: Add functions to manipulate CHAM_ipiv_t data structure at the runtime level

include/chameleon/runtime.h

@@ -10,7 +10,7 @@
  ***
  *
  * @brief The common runtimes API
- * @version 1.2.0
+ * @version 1.3.0
  * @author Mathieu Faverge
  * @author Cedric Augonnet
  * @author Cedric Castagnede
@@ -18,7 +18,7 @@
  * @author Samuel Thibault
  * @author Philippe Swartvagher
  * @author Matthieu Kuhn
- * @date 2022-02-22
+ * @date 2023-08-22
  *
  */
 #ifndef _chameleon_runtime_h_
@@ -705,6 +705,32 @@ void RUNTIME_ddisplay_oneprofile (cham_tasktype_t task);
 void RUNTIME_sdisplay_allprofile ();
 void RUNTIME_sdisplay_oneprofile (cham_tasktype_t task);
 
+void RUNTIME_ipiv_create ( CHAM_ipiv_t *ipiv );
+void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv );
+void RUNTIME_ipiv_init   ( CHAM_ipiv_t *ipiv );
+void RUNTIME_ipiv_gather ( CHAM_ipiv_t *desc, int *ipiv, int node );
+
+void *RUNTIME_ipiv_getaddr   ( CHAM_ipiv_t *ipiv, int m );
+void *RUNTIME_nextpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h );
+void *RUNTIME_prevpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h );
+
+static inline void *
+RUNTIME_pivot_getaddr( CHAM_ipiv_t *ipiv, int m, int h ) {
+    if ( h%2 == 0 ) {
+        return RUNTIME_nextpiv_getaddr( ipiv, m, -1 );
+    }
+    else {
+        return RUNTIME_prevpiv_getaddr( ipiv, m, -1 );
+    }
+}
+
+void RUNTIME_ipiv_flushk ( const RUNTIME_sequence_t *sequence,
+                           const CHAM_ipiv_t *ipiv, int m );
+void RUNTIME_ipiv_flush  ( const CHAM_ipiv_t *ipiv,
+                           const RUNTIME_sequence_t *sequence );
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+                           CHAM_ipiv_t *ws, int k, int h );
+
 /**
  * @}
  */

runtime/openmp/control/runtime_descriptor_ipiv.c

+/**
+ *
+ * @file openmp/runtime_descriptor_ipiv.c
+ *
+ * @copyright 2022-2023 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+ *                      Univ. Bordeaux. All rights reserved.
+ *
+ ***
+ *
+ * @brief Chameleon OpenMP descriptor routines
+ *
+ * @version 1.3.0
+ * @author Mathieu Faverge
+ * @author Matthieu Kuhn
+ * @date 2023-08-22
+ *
+ */
+#include "chameleon_openmp.h"
+
+void RUNTIME_ipiv_create( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void *RUNTIME_ipiv_getaddr( CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    return NULL;
+}
+
+void *RUNTIME_nextpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void *RUNTIME_prevpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
+                          const CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)sequence;
+    (void)ipiv;
+    (void)m;
+}
+
+void RUNTIME_ipiv_flush( const CHAM_ipiv_t        *ipiv,
+                         const RUNTIME_sequence_t *sequence )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)sequence;
+}
+
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+                           CHAM_ipiv_t *ipiv, int k, int h )
+{
+    assert( 0 );
+    (void)options;
+    (void)ipiv;
+    (void)k;
+    (void)h;
+}
+
+void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+{
+    assert( 0 );
+    (void)desc;
+    (void)ipiv;
+    (void)node;
+}

runtime/parsec/control/runtime_descriptor_ipiv.c

+/**
+ *
+ * @file parsec/runtime_descriptor_ipiv.c
+ *
+ * @copyright 2022-2023 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+ *                      Univ. Bordeaux. All rights reserved.
+ *
+ ***
+ *
+ * @brief Chameleon PaRSEC descriptor routines
+ *
+ * @version 1.3.0
+ * @author Mathieu Faverge
+ * @author Matthieu Kuhn
+ * @date 2023-08-22
+ *
+ */
+#include "chameleon_parsec.h"
+
+void RUNTIME_ipiv_create( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void *RUNTIME_ipiv_getaddr( CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    return NULL;
+}
+
+void *RUNTIME_nextpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void *RUNTIME_prevpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
+                          const CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)sequence;
+    (void)ipiv;
+    (void)m;
+}
+
+void RUNTIME_ipiv_flush( const CHAM_ipiv_t        *ipiv,
+                         const RUNTIME_sequence_t *sequence )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)sequence;
+}
+
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+                           CHAM_ipiv_t *ipiv, int k, int h )
+{
+    assert( 0 );
+    (void)options;
+    (void)ipiv;
+    (void)k;
+    (void)h;
+}
+
+void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+{
+    assert( 0 );
+    (void)desc;
+    (void)ipiv;
+    (void)node;
+}

runtime/quark/control/runtime_descriptor_ipiv.c

+/**
+ *
+ * @file quark/runtime_descriptor_ipiv.c
+ *
+ * @copyright 2022-2023 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+ *                      Univ. Bordeaux. All rights reserved.
+ *
+ ***
+ *
+ * @brief Chameleon Quark descriptor routines
+ *
+ * @version 1.3.0
+ * @author Mathieu Faverge
+ * @author Matthieu Kuhn
+ * @date 2023-08-22
+ *
+ */
+#include "chameleon_quark.h"
+
+void RUNTIME_ipiv_create( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void *RUNTIME_ipiv_getaddr( CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    return NULL;
+}
+
+void *RUNTIME_nextpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void *RUNTIME_prevpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)m;
+    (void)h;
+    return NULL;
+}
+
+void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
+                          const CHAM_ipiv_t *ipiv, int m )
+{
+    assert( 0 );
+    (void)sequence;
+    (void)ipiv;
+    (void)m;
+}
+
+void RUNTIME_ipiv_flush( const CHAM_ipiv_t        *ipiv,
+                         const RUNTIME_sequence_t *sequence )
+{
+    assert( 0 );
+    (void)ipiv;
+    (void)sequence;
+}
+
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+                           CHAM_ipiv_t *ipiv, int k, int h )
+{
+    assert( 0 );
+    (void)options;
+    (void)ipiv;
+    (void)k;
+    (void)h;
+}
+
+void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+{
+    assert( 0 );
+    (void)ipiv;
+}
+
+void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+{
+    assert( 0 );
+    (void)desc;
+    (void)ipiv;
+    (void)node;
+}

runtime/starpu/control/runtime_descriptor_ipiv.c

+/**
+ *
+ * @file starpu/runtime_descriptor_ipiv.c
+ *
+ * @copyright 2022-2023 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
+ *                      Univ. Bordeaux. All rights reserved.
+ *
+ ***
+ *
+ * @brief Chameleon StarPU descriptor routines
+ *
+ * @version 1.3.0
+ * @author Mathieu Faverge
+ * @author Matthieu Kuhn
+ * @date 2023-08-22
+ *
+ */
+#include "chameleon_starpu.h"
+
+/**
+ *  Create ws_pivot runtime structures
+ */
+void RUNTIME_ipiv_create( CHAM_ipiv_t *ipiv )
+{
+    assert( ipiv );
+
+    ipiv->ipiv    = (void*)calloc( ipiv->mt, sizeof(starpu_data_handle_t) );
+    ipiv->nextpiv = (void*)calloc( ipiv->mt, sizeof(starpu_data_handle_t) );
+    ipiv->prevpiv = (void*)calloc( ipiv->mt, sizeof(starpu_data_handle_t) );
+#if defined(CHAMELEON_USE_MPI)
+    /*
+     * Book the number of tags required to describe pivot structure
+     * One per handle type
+     */
+    {
+        chameleon_starpu_tag_init();
+        ipiv->mpitag_ipiv = chameleon_starpu_tag_book( (int64_t)(ipiv->mt) * 3 );
+        if ( ipiv->mpitag_ipiv == -1 ) {
+            chameleon_fatal_error("RUNTIME_ipiv_create", "Can't pursue computation since no more tags are available for ipiv structure");
+            return;
+        }
+        ipiv->mpitag_nextpiv = ipiv->mpitag_ipiv    + ipiv->mt;
+        ipiv->mpitag_prevpiv = ipiv->mpitag_nextpiv + ipiv->mt;
+    }
+#endif
+}
+
+/**
+ *  Destroy ws_pivot runtime structures
+ */
+void RUNTIME_ipiv_destroy( CHAM_ipiv_t *ipiv )
+{
+    int                   i;
+    starpu_data_handle_t *ipiv_handle    = (starpu_data_handle_t*)(ipiv->ipiv);
+    starpu_data_handle_t *nextpiv_handle = (starpu_data_handle_t*)(ipiv->nextpiv);
+    starpu_data_handle_t *prevpiv_handle = (starpu_data_handle_t*)(ipiv->prevpiv);
+
+    for(i=0; i<ipiv->mt; i++) {
+        if ( *ipiv_handle != NULL ) {
+            starpu_data_unregister( *ipiv_handle );
+            *ipiv_handle = NULL;
+        }
+        ipiv_handle++;
+
+        if ( *nextpiv_handle != NULL ) {
+            starpu_data_unregister( *nextpiv_handle );
+            *nextpiv_handle = NULL;
+        }
+        nextpiv_handle++;
+
+        if ( *prevpiv_handle != NULL ) {
+            starpu_data_unregister( *prevpiv_handle );
+            *prevpiv_handle = NULL;
+        }
+        prevpiv_handle++;
+    }
+
+    free( ipiv->ipiv    );
+    free( ipiv->nextpiv );
+    free( ipiv->prevpiv );
+    chameleon_starpu_tag_release( ipiv->mpitag_ipiv );
+}
+
+void *RUNTIME_ipiv_getaddr( CHAM_ipiv_t *ipiv, int m )
+{
+    starpu_data_handle_t *handle = (starpu_data_handle_t*)(ipiv->ipiv);
+    int64_t mm = m + (ipiv->i / ipiv->mb);
+
+    handle += mm;
+    assert( handle );
+
+    if ( *handle != NULL ) {
+        return *handle;
+    }
+
+    const CHAM_desc_t *A = ipiv->desc;
+    int owner = A->get_rankof( A, m, m );
+    int ncols = (mm == (ipiv->mt-1)) ? ipiv->m - mm * ipiv->mb : ipiv->mb;
+
+    starpu_vector_data_register( handle, -1, (uintptr_t)NULL, ncols, sizeof(int) );
+
+#if defined(CHAMELEON_USE_MPI)
+    {
+        int64_t tag = ipiv->mpitag_ipiv + mm;
+        starpu_mpi_data_register( *handle, tag, owner );
+    }
+#endif /* defined(CHAMELEON_USE_MPI) */
+
+    assert( *handle );
+    return *handle;
+}
+
+void *RUNTIME_nextpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    starpu_data_handle_t *nextpiv = (starpu_data_handle_t*)(ipiv->nextpiv);
+    int64_t mm = m + (ipiv->i / ipiv->mb);
+
+    nextpiv += mm;
+    assert( nextpiv );
+
+    if ( *nextpiv != NULL ) {
+        return *nextpiv;
+    }
+
+    const CHAM_desc_t *A = ipiv->desc;
+    int     owner = A->get_rankof( A, m, m );
+    int     ncols = (mm == (ipiv->mt-1)) ? ipiv->m - mm * ipiv->mb : ipiv->mb;
+    int64_t tag   = ipiv->mpitag_nextpiv + mm;
+
+    cppi_register( nextpiv, A->dtyp, ncols, tag, owner );
+
+    assert( *nextpiv );
+    return *nextpiv;
+}
+
+void *RUNTIME_prevpiv_getaddr( CHAM_ipiv_t *ipiv, int m, int h )
+{
+    starpu_data_handle_t *prevpiv = (starpu_data_handle_t*)(ipiv->prevpiv);
+    int64_t mm = m + (ipiv->i / ipiv->mb);
+
+    prevpiv += mm;
+    assert( prevpiv );
+
+    if ( *prevpiv != NULL ) {
+        return *prevpiv;
+    }
+
+    const CHAM_desc_t *A = ipiv->desc;
+    int     owner = A->get_rankof( A, m, m );
+    int     ncols = (mm == (ipiv->mt-1)) ? ipiv->m - mm * ipiv->mb : ipiv->mb;
+    int64_t tag   = ipiv->mpitag_prevpiv + mm;
+
+    cppi_register( prevpiv, A->dtyp, ncols, tag, owner );
+
+    assert( *prevpiv );
+    return *prevpiv;
+}
+
+void RUNTIME_ipiv_flushk( const RUNTIME_sequence_t *sequence,
+                          const CHAM_ipiv_t *ipiv, int m )
+{
+    starpu_data_handle_t *handle;
+    const CHAM_desc_t *A = ipiv->desc;
+    int64_t mm = m + ( ipiv->i / ipiv->mb );
+
+    handle = (starpu_data_handle_t*)(ipiv->nextpiv);
+    handle += mm;
+
+    if ( *handle != NULL ) {
+#if defined(CHAMELEON_USE_MPI)
+        starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+        if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
+#endif
+        {
+            chameleon_starpu_data_wont_use( *handle );
+        }
+    }
+
+    handle = (starpu_data_handle_t*)(ipiv->prevpiv);
+    handle += mm;
+
+    if ( *handle != NULL ) {
+#if defined(CHAMELEON_USE_MPI)
+        starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
+        if ( starpu_mpi_data_get_rank( *handle ) == A->myrank )
+#endif
+        {
+            chameleon_starpu_data_wont_use( *handle );
+        }
+    }
+
+    (void)sequence;
+    (void)ipiv;
+    (void)m;
+}
+
+void RUNTIME_ipiv_flush( const CHAM_ipiv_t        *ipiv,
+                         const RUNTIME_sequence_t *sequence )
+{
+    int m;
+
+    for (m = 0; m < ipiv->mt; m++)
+    {
+        RUNTIME_ipiv_flushk( sequence, ipiv, m );
+    }
+}
+
+void RUNTIME_ipiv_reducek( const RUNTIME_option_t *options,
+                           CHAM_ipiv_t *ipiv, int k, int h )
+{
+    starpu_data_handle_t nextpiv = RUNTIME_pivot_getaddr( ipiv, k, h   );
+    starpu_data_handle_t prevpiv = RUNTIME_pivot_getaddr( ipiv, k, h-1 );
+
+    if ( h < ipiv->n ) {
+#if defined(HAVE_STARPU_MPI_REDUX) && defined(CHAMELEON_USE_MPI)
+#if !defined(HAVE_STARPU_MPI_REDUX_WRAPUP)
+        starpu_mpi_redux_data_prio_tree( MPI_COMM_WORLD, nextpiv,
+                                         options->priority, 2 /* Binary tree */ );
+#endif
+#endif
+    }
+
+    /* Invalidate the previous pivot structure for correct initialization in later reuse */
+    if ( h > 0 ) {
+        starpu_data_invalidate_submit( prevpiv );
+    }
+
+    (void)options;
+}
+
+static void cl_ipiv_init_cpu_func(void *descr[], void *cl_arg)
+{
+    int *ipiv = (int *)STARPU_VECTOR_GET_PTR(descr[0]);
+
+#if !defined(CHAMELEON_SIMULATION)
+    {
+        int i, m0, n;
+        starpu_codelet_unpack_args( cl_arg, &m0, &n );
+
+        for( i=0; i<n; i++ ) {
+            ipiv[i] = m0 + i + 1;
+        }
+    }
+#endif
+}
+
+struct starpu_codelet cl_ipiv_init = {
+    .where     = STARPU_CPU,
+    .cpu_func  = cl_ipiv_init_cpu_func,
+    .nbuffers  = 1,
+};
+
+void RUNTIME_ipiv_init( CHAM_ipiv_t *ipiv )
+{
+    int64_t mt = ipiv->mt;
+    int64_t mb = ipiv->mb;
+    int     m;
+
+    for (m = 0; m < mt; m++) {
+        starpu_data_handle_t ipiv_src = RUNTIME_ipiv_getaddr( ipiv, m );
+        int m0 = m * mb;
+        int n  = (m == (mt-1)) ? ipiv->m - m0 : mb;
+
+        rt_starpu_insert_task(
+            &cl_ipiv_init,
+            STARPU_VALUE, &m0, sizeof(int),
+            STARPU_VALUE, &n,  sizeof(int),
+            STARPU_W, ipiv_src,
+            0);
+    }
+}
+
+void RUNTIME_ipiv_gather( CHAM_ipiv_t *desc, int *ipiv, int node )
+{
+    int64_t mt   = desc->mt;
+    int64_t mb   = desc->mb;
+    int64_t tag  = chameleon_starpu_tag_book( (int64_t)(desc->mt) );
+    int     rank = CHAMELEON_Comm_rank();
+    int     m;
+
+    for (m = 0; m < mt; m++, ipiv += mb) {
+        starpu_data_handle_t ipiv_src = RUNTIME_ipiv_getaddr( desc, m );
+
+#if defined(CHAMELEON_USE_MPI)
+        if ( (rank == node) ||
+             (rank == starpu_mpi_data_get_rank(ipiv_src)) )
+#endif
+        {
+            starpu_data_handle_t ipiv_dst;
+            int       ncols     = (m == (mt-1)) ? desc->m - m * mb : mb;
+            uintptr_t ipivptr   = (rank == node) ? (uintptr_t)ipiv : 0;
+            int       home_node = (rank == node) ? STARPU_MAIN_RAM : -1;
+
+            starpu_vector_data_register( &ipiv_dst, home_node, ipivptr, ncols, sizeof(int) );
+
+#if defined(CHAMELEON_USE_MPI)
+            starpu_mpi_data_register( ipiv_dst, tag + m, 0 );
+#endif /* defined(CHAMELEON_USE_MPI) */
+
+            assert( ipiv_dst );
+
+            starpu_data_cpy( ipiv_dst, ipiv_src, 0, NULL, NULL );
+            starpu_data_unregister( ipiv_dst );
+        }
+    }
+}