Commit ad9a910c authored by Mathieu Faverge's avatar Mathieu Faverge

Merge branch 'mpi_register' into 'master'

Add support for registering MPI data types

Closes #97

See merge request solverstack/chameleon!204
parents 80f20a19 4252942f
......@@ -101,9 +101,6 @@ set( RP_CHAMELEON_DICTIONNARY ${CMAKE_SOURCE_DIR}/cmake_modules/local_subs.py )
set( RP_CHAMELEON_PRECISIONS "s;d;c;z" )
include(RulesPrecisions)
option(CHAMELEON_USE_MIGRATE
"This options enables the data migration in QR algorithms" ON)
# Options to select the runtime
# -----------------------------
......@@ -175,6 +172,14 @@ if (CHAMELEON_ENABLE_MPI AND NOT CHAMELEON_USE_MPI)
message("-- ${BoldGreen}CHAMELEON_USE_MPI is set to OFF, turn it ON to use MPI (unsupported by Quark)${ColourReset}")
endif()
cmake_dependent_option(CHAMELEON_USE_MIGRATE
"This options enables the data migration in QR algorithms" ON
"CHAMELEON_USE_MPI" OFF)
cmake_dependent_option(CHAMELEON_USE_MPI_DATATYPES
"Enable MPI datatypes wheneve supported by the runtime" ON
"CHAMELEON_USE_MPI" OFF)
# Enable Cuda kernels if StarPU (only if StarPU or PaRSEC runtime is enabled)
# TODO: Default should be changed to ON/OFF when it will be ok
cmake_dependent_option(CHAMELEON_USE_CUDA
......@@ -656,6 +661,14 @@ if( CHAMELEON_SCHED_STARPU )
set(CHAMELEON_USE_MIGRATE "OFF")
message("-- ${Blue}CHAMELEON_USE_MIGRATE is turned OFF because starpu_mpi_data_migrate not found${ColourReset}")
endif()
check_function_exists(starpu_mpi_interface_datatype_register HAVE_STARPU_MPI_INTERFACE_DATATYPE_REGISTER)
if ( HAVE_STARPU_MPI_INTERFACE_DATATYPE_REGISTER )
message("-- ${Blue}Add definition HAVE_STARPU_MPI_INTERFACE_DATATYPE_REGISTER${ColourReset}")
else()
if( CHAMELEON_USE_MPI_DATATYPES )
set( CHAMELEON_USE_MPI_DATATYPES OFF )
endif()
endif()
check_function_exists(starpu_interface_copy2d HAVE_STARPU_INTERFACE_COPY2D)
if ( HAVE_STARPU_INTERFACE_COPY2D )
message("-- ${Blue}Add definition HAVE_STARPU_INTERFACE_COPY2D${ColourReset}")
......
......@@ -38,6 +38,10 @@
#if !defined(CHAMELEON_USE_MPI) && defined(CHAMELEON_USE_MIGRATE)
#undef CHAMELEON_USE_MIGRATE
#endif
#cmakedefine CHAMELEON_USE_MPI_DATATYPES
#if !defined(CHAMELEON_USE_MPI) && defined(CHAMELEON_USE_MPI_DATATYPES)
#undef CHAMELEON_USE_MPI_DATATYPES
#endif
/* GPU Support */
#cmakedefine CHAMELEON_USE_CUDA
......
......@@ -142,6 +142,8 @@ int RUNTIME_init( CHAM_context_t *chamctxt,
starpu_cublas_init();
#endif
starpu_cham_tile_interface_init();
return hres;
}
......@@ -157,6 +159,8 @@ void RUNTIME_finalize( CHAM_context_t *chamctxt )
return;
}
starpu_cham_tile_interface_fini();
#if defined(CHAMELEON_USE_CUDA) && !defined(CHAMELEON_SIMULATION)
starpu_cublas_shutdown();
#endif
......
......@@ -34,7 +34,7 @@ struct starpu_cham_tile_interface_s
cham_flttype_t flttype; /**< Type of the elements of the matrix */
size_t allocsize; /**< size actually currently allocated */
size_t tilesize; /**< size of the elements of the matrix */
CHAM_tile_t tile; /**< Internal tile structure used to store
CHAM_tile_t tile; /**< Internal tile structure used to store
information on non memory home_node */
};
......@@ -53,4 +53,7 @@ cti_interface_get( starpu_cham_tile_interface_t *interface )
return &(interface->tile);
}
void starpu_cham_tile_interface_init();
void starpu_cham_tile_interface_fini();
#endif /* _cham_tile_interface_h_ */
......@@ -38,6 +38,11 @@
#cmakedefine HAVE_STARPU_MPI_COMM_GET_ATTR
#cmakedefine HAVE_STARPU_MPI_INIT_CONF
#cmakedefine HAVE_STARPU_MPI_WAIT_FOR_ALL
#cmakedefine HAVE_STARPU_MPI_INTERFACE_DATATYPE_REGISTER
#if !defined(HAVE_STARPU_MPI_INTERFACE_DATATYPE_REGISTER) && defined(CHAMELEON_USE_MPI_DATATYPES)
#error "This version of StarPU does not support MPI datatypes (Please compile with -DCHAMELEON_USE_MPI_DATATYPES=OFF)"
#endif
#if defined(CHAMELEON_USE_MPI)
#include <starpu_mpi.h>
......
......@@ -319,30 +319,47 @@ cti_unpack_data( starpu_data_handle_t handle, unsigned node, void *ptr, size_t c
starpu_cham_tile_interface_t *cham_tile_interface = (starpu_cham_tile_interface_t *)
starpu_data_get_interface_on_node(handle, node);
CHAM_tile_t dsttile;
char *tmp = ptr;
/* Extract the size of the information t unpack */
memcpy( &(cham_tile_interface->allocsize), tmp, sizeof(size_t) );
tmp += sizeof(size_t);
#if defined(CHAMELEON_USE_MPI_DATATYPES)
/*
* We may end up here if an early reception occured before the handle of the
* received data has been registered. Thus, datatype was not existant and we
* need to unpack the data ourselves
*/
STARPU_ASSERT( count == cham_tile_interface->allocsize );
STARPU_ASSERT( cham_tile_interface->tile.format & CHAMELEON_TILE_FULLRANK );
#else
{
CHAM_tile_t dsttile;
/* Extract the size of the information to unpack */
memcpy( &(cham_tile_interface->allocsize), tmp, sizeof(size_t) );
tmp += sizeof(size_t);
/* Extract the tile metadata of the remote tile */
memcpy( &dsttile, tmp, sizeof(CHAM_tile_t) );
tmp += sizeof(CHAM_tile_t);
/* Extract the tile metadata of the remote tile */
memcpy( &dsttile, tmp, sizeof(CHAM_tile_t) );
tmp += sizeof(CHAM_tile_t);
cham_tile_interface->tile.format = dsttile.format;
cham_tile_interface->tile.ld = cham_tile_interface->tile.m;
STARPU_ASSERT( cham_tile_interface->tile.m == dsttile.m );
STARPU_ASSERT( cham_tile_interface->tile.n == dsttile.n );
STARPU_ASSERT( count == cham_tile_interface->allocsize + sizeof(size_t) + sizeof(CHAM_tile_t) );
/*
* Update with the local information. Data is packed now, and do not
* need leading dimension anymore
*/
cham_tile_interface->tile.format = dsttile.format;
cham_tile_interface->tile.ld = cham_tile_interface->tile.m;
STARPU_ASSERT( cham_tile_interface->tile.m == dsttile.m );
STARPU_ASSERT( cham_tile_interface->tile.n == dsttile.n );
STARPU_ASSERT( count == cham_tile_interface->allocsize + sizeof(size_t) + sizeof(CHAM_tile_t) );
}
#endif
/* Unpack the real data */
if ( cham_tile_interface->tile.format & CHAMELEON_TILE_FULLRANK ) {
cti_unpack_data_fullrank( cham_tile_interface, tmp );
}
else {
STARPU_ASSERT_MSG( 1, "Unsupported format for pack." );
STARPU_ASSERT_MSG( 1, "Unsupported format for unpack." );
}
/* Free the received information */
......@@ -481,8 +498,36 @@ cti_handle_get_allocsize( starpu_data_handle_t handle )
return cham_tile_interface->allocsize;
}
#if defined(CHAMELEON_USE_MPI_DATATYPES)
int
cti_allocate_datatype( starpu_data_handle_t handle,
MPI_Datatype *datatype )
{
int ret;
starpu_cham_tile_interface_t *cham_tile_interface = (starpu_cham_tile_interface_t *)
starpu_data_get_interface_on_node( handle, STARPU_MAIN_RAM );
size_t m = cham_tile_interface->tile.m;
size_t n = cham_tile_interface->tile.n;
size_t ld = cham_tile_interface->tile.ld;
size_t elemsize = CHAMELEON_Element_Size( cham_tile_interface->flttype );
ret = MPI_Type_vector( n, m * elemsize, ld * elemsize, MPI_BYTE, datatype );
STARPU_ASSERT_MSG(ret == MPI_SUCCESS, "MPI_Type_vector failed");
ret = MPI_Type_commit( datatype );
STARPU_ASSERT_MSG(ret == MPI_SUCCESS, "MPI_Type_commit failed");
return 0;
}
void
starpu_cham_tile_interface_init() __attribute__((constructor));
cti_free_datatype( MPI_Datatype *datatype )
{
MPI_Type_free( datatype );
}
#endif
void
starpu_cham_tile_interface_init()
......@@ -490,5 +535,21 @@ starpu_cham_tile_interface_init()
if ( starpu_interface_cham_tile_ops.interfaceid == STARPU_UNKNOWN_INTERFACE_ID )
{
starpu_interface_cham_tile_ops.interfaceid = starpu_data_interface_get_next_id();
#if defined(CHAMELEON_USE_MPI_DATATYPES)
starpu_mpi_interface_datatype_register( starpu_interface_cham_tile_ops.interfaceid,
cti_allocate_datatype,
cti_free_datatype );
#endif
}
}
void
starpu_cham_tile_interface_fini()
{
if ( starpu_interface_cham_tile_ops.interfaceid != STARPU_UNKNOWN_INTERFACE_ID )
{
#if defined(CHAMELEON_USE_MPI_DATATYPES)
starpu_mpi_interface_datatype_unregister( starpu_interface_cham_tile_ops.interfaceid );
#endif
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment