runtime_descriptor.c 13.5 KB
Newer Older
1
/**
2
 *
3
 * @file starpu/runtime_descriptor.c
4
 *
5
 * @copyright 2009-2014 The University of Tennessee and The University of
Mathieu Faverge's avatar
Mathieu Faverge committed
6
 *                      Tennessee Research Foundation. All rights reserved.
7
 * @copyright 2012-2019 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
8
 *                      Univ. Bordeaux. All rights reserved.
9
 *
10 11 12
 ***
 *
 * @brief Chameleon StarPU descriptor routines
13
 *
Mathieu Faverge's avatar
Mathieu Faverge committed
14
 * @version 1.0.0
15 16 17 18 19
 * @author Cedric Augonnet
 * @author Mathieu Faverge
 * @author Cedric Castagnede
 * @date 2010-11-15
 *
20
 */
21
#include <stdlib.h>
22
#include <unistd.h>
23
#include "chameleon_starpu.h"
24

25
/**
26
 *  Set the tag sizes
27
 */
28
#if defined(CHAMELEON_USE_MPI)
29

30
/* Take 24 bits for the tile id, and 7 bits for descriptor id.
31
 These values can be changed through the call CHAMELEON_user_tag_size(int tag_width, int tag_sep) */
32 33
#define TAG_WIDTH_MIN 20
static int tag_width = 31;
34
static int tag_sep   = 20;
35
static int _tag_mpi_initialized_ = 0;
36

Mathieu Faverge's avatar
Mathieu Faverge committed
37 38 39 40 41 42
static inline int
chameleon_starpu_tag_init( int user_tag_width,
                           int user_tag_sep )
{
    if (!_tag_mpi_initialized_) {
        int ok = 0;
43
        uintptr_t tag_ub;
Mathieu Faverge's avatar
Mathieu Faverge committed
44 45 46 47

        tag_width = user_tag_width;
        tag_sep   = user_tag_sep;

48
        void *tag_ub_p = NULL;
Mathieu Faverge's avatar
Mathieu Faverge committed
49
#if defined(HAVE_STARPU_MPI_COMM_GET_ATTR)
50
        starpu_mpi_comm_get_attr(MPI_COMM_WORLD, STARPU_MPI_TAG_UB, &tag_ub_p, &ok);
Mathieu Faverge's avatar
Mathieu Faverge committed
51
#else
52
        MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &tag_ub_p, &ok);
Mathieu Faverge's avatar
Mathieu Faverge committed
53
#endif
54
        tag_ub = (uintptr_t)tag_ub_p;
Mathieu Faverge's avatar
Mathieu Faverge committed
55 56

        if ( !ok ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
57
            chameleon_error("RUNTIME_desc_create", "MPI_TAG_UB not known by StarPU");
Mathieu Faverge's avatar
Mathieu Faverge committed
58 59
        }

60
        while ( ((uintptr_t)((1UL<<tag_width) - 1) > tag_ub ) &&
Mathieu Faverge's avatar
Mathieu Faverge committed
61 62 63 64 65 66 67
                (tag_width >= TAG_WIDTH_MIN) )
        {
            tag_width--;
            tag_sep--;
        }

        if ( tag_width < TAG_WIDTH_MIN ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
68
            chameleon_error("RUNTIME_desc_create", "MPI_TAG_UB too small to identify all the data");
69
            return CHAMELEON_ERR_OUT_OF_RESOURCES;
Mathieu Faverge's avatar
Mathieu Faverge committed
70 71 72
        }

        _tag_mpi_initialized_ = 1;
73
        return CHAMELEON_SUCCESS;
Mathieu Faverge's avatar
Mathieu Faverge committed
74 75
    }
    else {
76
        return CHAMELEON_ERR_REINITIALIZED;
Mathieu Faverge's avatar
Mathieu Faverge committed
77 78 79 80
    }
}


81
#ifndef HAVE_STARPU_MPI_DATA_REGISTER
82 83 84 85
#define starpu_mpi_data_register( handle_, tag_, owner_ )       \
    do {                                                        \
        starpu_data_set_rank( (handle_), (owner_) );            \
        starpu_data_set_tag( (handle_), (tag_) );               \
86 87 88 89 90
    } while(0)
#endif

#endif

91 92 93
void RUNTIME_comm_set_tag_sizes( int user_tag_width,
                                 int user_tag_sep )
{
THIBAULT Samuel's avatar
THIBAULT Samuel committed
94
#if defined(CHAMELEON_USE_MPI)
Mathieu Faverge's avatar
Mathieu Faverge committed
95 96
    int rc;
    rc = chameleon_starpu_tag_init( user_tag_width, user_tag_sep );
97
    if ( rc != CHAMELEON_SUCCESS ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
98
        chameleon_error("RUNTIME_user_tag_size",
99
                    "must be called before creating any Cham descriptor with CHAMELEON_Desc_create(). The tag sizes will not be modified.");
100
    }
THIBAULT Samuel's avatar
THIBAULT Samuel committed
101
#endif
102
    (void)user_tag_width; (void)user_tag_sep;
103 104
}

105
/**
106
 *  Malloc/Free of the data
107
 */
108 109 110 111 112
#ifdef STARPU_MALLOC_SIMULATION_FOLDED
#define FOLDED STARPU_MALLOC_SIMULATION_FOLDED
#else
#define FOLDED 0
#endif
113

114
void *RUNTIME_malloc( size_t size )
115 116 117 118
{
#if defined(CHAMELEON_SIMULATION) && !defined(STARPU_MALLOC_SIMULATION_FOLDED) && !defined(CHAMELEON_USE_MPI)
    return (void*) 1;
#else
119
    void *ptr;
120

121
    if (starpu_malloc_flags(&ptr, size, STARPU_MALLOC_PINNED|FOLDED|STARPU_MALLOC_COUNT) != 0) {
122
        return NULL;
123 124
    }
    return ptr;
125 126 127
#endif
}

128 129
void RUNTIME_free( void  *ptr,
                   size_t size )
130 131
{
#if defined(CHAMELEON_SIMULATION) && !defined(STARPU_MALLOC_SIMULATION_FOLDED) && !defined(CHAMELEON_USE_MPI)
132
    (void)ptr; (void)size;
THIBAULT Samuel's avatar
THIBAULT Samuel committed
133
    return;
134
#else
135
    starpu_free_flags(ptr, size, STARPU_MALLOC_PINNED|FOLDED|STARPU_MALLOC_COUNT);
136 137 138
#endif
}

139
/**
140
 *  Create data descriptor
141
 */
142
void RUNTIME_desc_create( CHAM_desc_t *desc )
143 144 145 146 147 148 149
{
    int64_t lmt = desc->lmt;
    int64_t lnt = desc->lnt;

    desc->occurences = 1;

    /*
150 151
     * Allocate starpu_handle_t array (handlers are initialized on the fly when
     * discovered by any algorithm to save space)
152 153 154 155
     */
    desc->schedopt = (void*)calloc(lnt*lmt,sizeof(starpu_data_handle_t));
    assert(desc->schedopt);

156
#if defined(CHAMELEON_USE_CUDA) && !defined(CHAMELEON_SIMULATION)
157 158 159 160 161
    /*
     * Register allocated memory as CUDA pinned memory
     */
    if ( (desc->use_mat == 1) && (desc->register_mat == 1) )
    {
162
        int64_t eltsze = CHAMELEON_Element_Size(desc->dtyp);
163 164
        size_t size = (size_t)(desc->llm) * (size_t)(desc->lln) * eltsze;
        cudaError_t rc;
165

166 167 168 169
        /* Register the matrix as pinned memory */
        rc = cudaHostRegister( desc->mat, size, cudaHostRegisterPortable );
        if ( rc != cudaSuccess )
        {
170 171
            /* Disable the unregister as register failed */
            desc->register_mat = 0;
Mathieu Faverge's avatar
Mathieu Faverge committed
172
            chameleon_warning("RUNTIME_desc_create(StarPU): cudaHostRegister - ", cudaGetErrorString( rc ));
173 174 175
        }
    }
#endif
176

177
    if (desc->ooc) {
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
        char   *backend = getenv("STARPU_DISK_SWAP_BACKEND");

        if (backend && strcmp(backend, "unistd_o_direct") == 0) {
            int     lastmm   = desc->lm - (desc->lmt-1) * desc->mb;
            int     lastnn   = desc->ln - (desc->lnt-1) * desc->nb;
            int64_t eltsze   = CHAMELEON_Element_Size(desc->dtyp);
            int     pagesize = getpagesize();

            if ( ((desc->mb * desc->nb * eltsze) % pagesize != 0) ||
                 ((lastmm   * desc->nb * eltsze) % pagesize != 0) ||
                 ((desc->mb * lastnn   * eltsze) % pagesize != 0) ||
                 ((lastmm   * lastnn   * eltsze) % pagesize != 0) )
            {
                chameleon_error("RUNTIME_desc_create", "Matrix and tile size not suitable for out-of-core: all tiles have to be multiples of the system page size. Tip : choose 'n' and 'nb' as both multiples of 32.");
                return;
            }
194 195
        }
    }
196

197
#if defined(CHAMELEON_USE_MPI)
198 199 200 201
    /*
     * Check that we are not going over MPI tag limitations
     */
    {
Mathieu Faverge's avatar
Mathieu Faverge committed
202
        chameleon_starpu_tag_init( tag_width, tag_sep );
203 204

        /* Check that we won't create overflow in tags used */
205
        if ( ((uintptr_t)(lnt*lmt)) > ((uintptr_t)(1UL<<tag_sep)) ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
206
            chameleon_fatal_error("RUNTIME_desc_create", "Too many tiles in the descriptor for MPI tags");
207 208 209 210
            return;
        }
        assert(lmt*lmt<=(1<<tag_sep));

211
        if ( ((uintptr_t)desc->id) >= (uintptr_t)(1UL<<(tag_width-tag_sep)) ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
212
            chameleon_fatal_error("RUNTIME_desc_create", "Number of descriptor available in MPI mode out of stock");
213 214
            return;
        }
215
        assert( ((uintptr_t)desc->id) < (uintptr_t)(1UL<<(tag_width-tag_sep)) );
216 217 218 219
    }
#endif
}

220
/**
221
 *  Destroy data descriptor
222
 */
223
void RUNTIME_desc_destroy( CHAM_desc_t *desc )
224 225 226 227 228
{
    desc->occurences--;

    /*
     * If this is the last descriptor using the matrix, we release the handle
229
     * and unregister the GPU data
230 231 232 233 234 235 236
     */
    if ( desc->occurences == 0 ) {
        starpu_data_handle_t *handle = (starpu_data_handle_t*)(desc->schedopt);
        int lmt = desc->lmt;
        int lnt = desc->lnt;
        int m, n;

237
        for (n = 0; n < lnt; n++) {
238 239
            for (m = 0; m < lmt; m++)
            {
240 241
                if (*handle != NULL) {
                    starpu_data_unregister(*handle);
242 243 244
                }
                handle++;
            }
245
        }
246

247
#if defined(CHAMELEON_USE_CUDA) && !defined(CHAMELEON_SIMULATION)
248 249
        if ( (desc->use_mat == 1) && (desc->register_mat == 1) )
        {
250 251 252
            /* Unmap the pinned memory associated to the matrix */
            if (cudaHostUnregister(desc->mat) != cudaSuccess)
            {
Mathieu Faverge's avatar
Mathieu Faverge committed
253
                chameleon_warning("RUNTIME_desc_destroy(StarPU)",
254 255
                                  "cudaHostUnregister failed to unregister the "
                                  "pinned memory associated to the matrix");
256 257
            }
        }
258
#endif /* defined(CHAMELEON_USE_CUDA) */
259 260 261 262 263

        free(desc->schedopt);
    }
}

264
/**
265
 *  Acquire data
266
 */
267
int RUNTIME_desc_acquire( const CHAM_desc_t *desc )
268
{
269 270 271 272
    starpu_data_handle_t *handle = (starpu_data_handle_t*)(desc->schedopt);
    int lmt = desc->lmt;
    int lnt = desc->lnt;
    int m, n;
273

274 275 276 277
    for (n = 0; n < lnt; n++) {
        for (m = 0; m < lmt; m++)
        {
            if ( (*handle == NULL) ||
Mathieu Faverge's avatar
Mathieu Faverge committed
278
                 !chameleon_desc_islocal( desc, m, n ) )
279 280 281 282 283 284 285 286
            {
                handle++;
                continue;
            }
            starpu_data_acquire(*handle, STARPU_R);
            handle++;
        }
    }
287
    return CHAMELEON_SUCCESS;
288 289
}

290
/**
291
 *  Release data
292
 */
293
int RUNTIME_desc_release( const CHAM_desc_t *desc )
294 295 296 297 298 299
{
    starpu_data_handle_t *handle = (starpu_data_handle_t*)(desc->schedopt);
    int lmt = desc->lmt;
    int lnt = desc->lnt;
    int m, n;

300
    for (n = 0; n < lnt; n++) {
301 302 303
        for (m = 0; m < lmt; m++)
        {
            if ( (*handle == NULL) ||
Mathieu Faverge's avatar
Mathieu Faverge committed
304
                 !chameleon_desc_islocal( desc, m, n ) )
305 306 307 308
            {
                handle++;
                continue;
            }
309
            starpu_data_release(*handle);
310 311
            handle++;
        }
312
    }
313
    return CHAMELEON_SUCCESS;
314 315
}

316
/**
317
 *  Flush cached data
318
 */
319 320 321 322 323 324 325
void RUNTIME_flush()
{
#if defined(CHAMELEON_USE_MPI)
    starpu_mpi_cache_flush_all_data(MPI_COMM_WORLD);
#endif
}

326
/**
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
 * Different implementations of the flush call based on StarPU version
 */
#ifdef HAVE_STARPU_DATA_WONT_USE

static inline void
chameleon_starpu_data_wont_use( starpu_data_handle_t handle ) {
    starpu_data_wont_use( handle );
}

#elif defined HAVE_STARPU_IDLE_PREFETCH

static inline void
chameleon_starpu_data_flush( starpu_data_handle_t handle)
{
    starpu_data_idle_prefetch_on_node(handle, STARPU_MAIN_RAM, 1);
    starpu_data_release_on_node(handle, -1);
}

static inline void
chameleon_starpu_data_wont_use( starpu_data_handle_t handle ) {
    starpu_data_acquire_on_node_cb( handle, -1, STARPU_R,
                                    chameleon_starpu_data_flush, handle );
}

#else

static inline void
chameleon_starpu_data_wont_use( starpu_data_handle_t handle ) {
    starpu_data_acquire_cb( handle, STARPU_R,
                            (void (*)(void*))&starpu_data_release, handle );
357
}
358 359 360

#endif

361 362
void RUNTIME_desc_flush( const CHAM_desc_t     *desc,
                         const RUNTIME_sequence_t *sequence )
363 364 365 366 367 368
{
    starpu_data_handle_t *handle = (starpu_data_handle_t*)(desc->schedopt);
    int lmt = desc->lmt;
    int lnt = desc->lnt;
    int m, n;

369 370 371
    for (n = 0; n < lnt; n++)
    {
        for (m = 0; m < lmt; m++, handle++)
372
        {
373
            if ( *handle == NULL ) {
374 375
                continue;
            }
376

377 378 379
#if defined(CHAMELEON_USE_MPI)
            starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
#endif
Mathieu Faverge's avatar
Mathieu Faverge committed
380
            if ( chameleon_desc_islocal( desc, m, n ) ) {
381 382
                chameleon_starpu_data_wont_use( *handle );
            }
383
        }
384
    }
385 386

    (void)sequence;
387 388
}

389 390
void RUNTIME_data_flush( const RUNTIME_sequence_t *sequence,
                         const CHAM_desc_t *A, int m, int n )
391
{
Mathieu Faverge's avatar
Mathieu Faverge committed
392 393
    int64_t mm = m + (A->i / A->mb);
    int64_t nn = n + (A->j / A->nb);
Mathieu Faverge's avatar
Mathieu Faverge committed
394

Mathieu Faverge's avatar
Mathieu Faverge committed
395 396
    starpu_data_handle_t *handle = A->schedopt;
    handle += ((int64_t)A->lmt) * nn + mm;
397

398 399
    if (*handle == NULL) {
        return;
400
    }
THIBAULT Samuel's avatar
THIBAULT Samuel committed
401

402 403 404
#if defined(CHAMELEON_USE_MPI)
    starpu_mpi_cache_flush( MPI_COMM_WORLD, *handle );
#endif
405

Mathieu Faverge's avatar
Mathieu Faverge committed
406
    if ( chameleon_desc_islocal( A, m, n ) ) {
407
        chameleon_starpu_data_wont_use( *handle );
408
    }
409

410
    (void)sequence;
411 412
}

413
#if defined(CHAMELEON_USE_MIGRATE)
414 415
void RUNTIME_data_migrate( const RUNTIME_sequence_t *sequence,
                           const CHAM_desc_t *A, int Am, int An, int new_rank )
416
{
417
#if defined(HAVE_STARPU_MPI_DATA_MIGRATE)
418 419 420 421 422 423 424 425 426 427 428 429 430
    starpu_data_handle_t *handle = (starpu_data_handle_t*)(A->schedopt);
    starpu_data_handle_t lhandle;
    handle += ((int64_t)(A->lmt) * (int64_t)An + (int64_t)Am);

    lhandle = *handle;
    if ( lhandle == NULL ) {
        /* Register the data */
        lhandle = RUNTIME_data_getaddr( A, Am, An );
    }

    starpu_mpi_data_migrate( MPI_COMM_WORLD, lhandle, new_rank );

    (void)sequence;
431 432 433
#else
    (void)sequence; (void)A; (void)Am; (void)An; (void)new_rank;
#endif
434 435 436
}
#endif

437
/**
438
 *  Get data addr
439
 */
440 441 442 443 444
/* For older revision of StarPU, STARPU_MAIN_RAM is not defined */
#ifndef STARPU_MAIN_RAM
#define STARPU_MAIN_RAM 0
#endif

445
void *RUNTIME_data_getaddr( const CHAM_desc_t *A, int m, int n )
446
{
Mathieu Faverge's avatar
Mathieu Faverge committed
447 448
    int64_t mm = m + (A->i / A->mb);
    int64_t nn = n + (A->j / A->nb);
449

Mathieu Faverge's avatar
Mathieu Faverge committed
450 451
    starpu_data_handle_t *ptrtile = A->schedopt;
    ptrtile += ((int64_t)A->lmt) * nn + mm;
452 453

    if (*ptrtile == NULL) {
454 455
        int home_node = -1;
        void *user_ptr = NULL;
Mathieu Faverge's avatar
Mathieu Faverge committed
456 457
        int myrank = A->myrank;
        int owner  = A->get_rankof( A, m, n );
458
        int64_t eltsze = CHAMELEON_Element_Size(A->dtyp);
Mathieu Faverge's avatar
Mathieu Faverge committed
459 460
        int tempmm = (mm == A->lmt-1) ? (A->lm - mm * A->mb) : A->mb;
        int tempnn = (nn == A->lnt-1) ? (A->ln - nn * A->nb) : A->nb;
461 462

        if ( myrank == owner ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
463
            user_ptr = A->get_blkaddr(A, m, n);
464 465
            if ( user_ptr != NULL ) {
                home_node = STARPU_MAIN_RAM;
466
            }
467 468
        }

469
        starpu_matrix_data_register( ptrtile, home_node, (uintptr_t) user_ptr,
Mathieu Faverge's avatar
Mathieu Faverge committed
470
                                     BLKLDD(A, m),
471
                                     tempmm, tempnn, eltsze );
472

473 474 475 476 477 478 479
#if defined(HAVE_STARPU_DATA_SET_OOC_FLAG)
        if ( A->ooc == 0 ) {
            starpu_data_set_ooc_flag( *ptrtile, 0 );
        }
#endif

#if defined(HAVE_STARPU_DATA_SET_COORDINATES)
480
        starpu_data_set_coordinates( *ptrtile, 2, m, n );
481 482
#endif

483
#if defined(CHAMELEON_USE_MPI)
484
        {
Mathieu Faverge's avatar
Mathieu Faverge committed
485 486
            int64_t block_ind = A->lmt * nn + mm;
            starpu_mpi_data_register(*ptrtile, (A->id << tag_sep) | (block_ind), owner);
487
        }
488
#endif /* defined(CHAMELEON_USE_MPI) */
489 490
    }

491
    return *ptrtile;
492
}