runtime_descriptor.c 11.6 KB
Newer Older
Mathieu Faverge's avatar
Mathieu Faverge committed
1
/**
2 3
 *
 * @file runtime_descriptor.c
Mathieu Faverge's avatar
Mathieu Faverge committed
4
 *
Mathieu Faverge's avatar
Mathieu Faverge committed
5
 * @copyright 2012-2017 The University of Tennessee and The University of
Mathieu Faverge's avatar
Mathieu Faverge committed
6
 *                      Tennessee Research Foundation. All rights reserved.
Mathieu Faverge's avatar
Mathieu Faverge committed
7 8 9
 * @copyright 2012-2017 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria,
 *                      Univ. Bordeaux. All rights reserved.
 *
Mathieu Faverge's avatar
Mathieu Faverge committed
10 11 12 13
 ***
 *
 * @brief Chameleon PaRSEC descriptor routines
 *
Mathieu Faverge's avatar
Mathieu Faverge committed
14 15 16 17
 * @version 1.0.0
 * @author Reazul Hoque
 * @author Mathieu Faverge
 * @date 2017-01-12
Mathieu Faverge's avatar
Mathieu Faverge committed
18
 *
19
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
20
#include <stdlib.h>
21
#include "chameleon_parsec.h"
Mathieu Faverge's avatar
Mathieu Faverge committed
22 23 24 25
#include <parsec/data.h>
#include <parsec/datatype.h>
#include <parsec/arena.h>

26 27 28 29 30 31 32 33 34 35 36 37
/* Variable parsec_dtd_no_of_arenas is private and cannot be changed */
#define MORSE_PARSEC_DTD_NO_OF_ARENA 16 /**< Number of arenas available per DTD */

typedef struct morse_parsec_arena_s {
    /* int mb; */
    /* int nb; */
    /* MORSE_enum dtype; */
    size_t size;
} morse_parsec_arena_t;

static int morse_parsec_nb_arenas = 0;
static morse_parsec_arena_t morse_parsec_registered_arenas[MORSE_PARSEC_DTD_NO_OF_ARENA] = { { 0 } };
Mathieu Faverge's avatar
Mathieu Faverge committed
38

Mathieu Faverge's avatar
Mathieu Faverge committed
39 40 41 42 43
void RUNTIME_comm_set_tag_sizes( int user_tag_width,
                                 int user_tag_sep )
{
    (void)user_tag_width;
    (void)user_tag_sep;
44 45
}

Mathieu Faverge's avatar
Mathieu Faverge committed
46
void *RUNTIME_malloc( size_t size )
47 48 49 50
{
    return malloc(size);
}

Mathieu Faverge's avatar
Mathieu Faverge committed
51
void RUNTIME_free( void *ptr, size_t size )
52 53
{
    (void)size;
Mathieu Faverge's avatar
Mathieu Faverge committed
54
    free(ptr);
55 56 57
    return;
}

Mathieu Faverge's avatar
Mathieu Faverge committed
58
static inline void
Mathieu Faverge's avatar
Mathieu Faverge committed
59
morse_parsec_key_to_coordinates(parsec_data_collection_t *data_collection, parsec_data_key_t key,
Mathieu Faverge's avatar
Mathieu Faverge committed
60 61
                                int *m, int *n)
{
Mathieu Faverge's avatar
Mathieu Faverge committed
62
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
63 64 65 66 67 68 69 70 71
    MORSE_desc_t *mdesc = pdesc->desc;
    int _m, _n;

    _m = key % mdesc->lmt;
    _n = key / mdesc->lmt;
    *m = _m - mdesc->i / mdesc->mb;
    *n = _n - mdesc->j / mdesc->nb;
}

Mathieu Faverge's avatar
Mathieu Faverge committed
72
static inline parsec_data_key_t
Mathieu Faverge's avatar
Mathieu Faverge committed
73
morse_parsec_data_key(parsec_data_collection_t *data_collection, ...)
Mathieu Faverge's avatar
Mathieu Faverge committed
74
{
Mathieu Faverge's avatar
Mathieu Faverge committed
75
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
76 77 78 79 80
    MORSE_desc_t *mdesc = pdesc->desc;
    va_list ap;
    int m, n;

    /* Get coordinates */
Mathieu Faverge's avatar
Mathieu Faverge committed
81
    va_start(ap, data_collection);
Mathieu Faverge's avatar
Mathieu Faverge committed
82 83 84 85 86 87 88 89 90 91 92
    m = va_arg(ap, unsigned int);
    n = va_arg(ap, unsigned int);
    va_end(ap);

    /* Offset by (i,j) to translate (m,n) in the global matrix */
    m += mdesc->i / mdesc->mb;
    n += mdesc->j / mdesc->nb;

    return ((n * mdesc->lmt) + m);
}

Mathieu Faverge's avatar
Mathieu Faverge committed
93
static inline uint32_t
Mathieu Faverge's avatar
Mathieu Faverge committed
94
morse_parsec_rank_of(parsec_data_collection_t *data_collection, ...)
Mathieu Faverge's avatar
Mathieu Faverge committed
95
{
Mathieu Faverge's avatar
Mathieu Faverge committed
96
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
97 98 99 100 101
    MORSE_desc_t *mdesc = pdesc->desc;
    va_list ap;
    int m, n;

    /* Get coordinates */
Mathieu Faverge's avatar
Mathieu Faverge committed
102
    va_start(ap, data_collection);
Mathieu Faverge's avatar
Mathieu Faverge committed
103 104 105 106 107 108 109 110 111 112 113
    m = va_arg(ap, unsigned int);
    n = va_arg(ap, unsigned int);
    va_end(ap);

    /* Offset by (i,j) to translate (m,n) in the global matrix */
    m += mdesc->i / mdesc->mb;
    n += mdesc->j / mdesc->nb;

    return mdesc->get_rankof( mdesc, m, n );
}

Mathieu Faverge's avatar
Mathieu Faverge committed
114
static inline uint32_t
Mathieu Faverge's avatar
Mathieu Faverge committed
115
morse_parsec_rank_of_key(parsec_data_collection_t *data_collection, parsec_data_key_t key)
Mathieu Faverge's avatar
Mathieu Faverge committed
116 117
{
    int m, n;
Mathieu Faverge's avatar
Mathieu Faverge committed
118 119
    morse_parsec_key_to_coordinates(data_collection, key, &m, &n);
    return morse_parsec_rank_of(data_collection, m, n);
Mathieu Faverge's avatar
Mathieu Faverge committed
120 121
}

Mathieu Faverge's avatar
Mathieu Faverge committed
122
static inline int32_t
123
morse_parsec_vpid_of(parsec_data_collection_t *data_collection, ... )
Mathieu Faverge's avatar
Mathieu Faverge committed
124
{
125
    (void)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
126 127 128
    return 0;
}

Mathieu Faverge's avatar
Mathieu Faverge committed
129
static inline int32_t
Mathieu Faverge's avatar
Mathieu Faverge committed
130
morse_parsec_vpid_of_key(parsec_data_collection_t *data_collection, parsec_data_key_t key)
Mathieu Faverge's avatar
Mathieu Faverge committed
131 132
{
    int m, n;
Mathieu Faverge's avatar
Mathieu Faverge committed
133 134
    morse_parsec_key_to_coordinates(data_collection, key, &m, &n);
    return morse_parsec_vpid_of(data_collection, m, n);
Mathieu Faverge's avatar
Mathieu Faverge committed
135 136
}

Mathieu Faverge's avatar
Mathieu Faverge committed
137
static inline parsec_data_t*
Mathieu Faverge's avatar
Mathieu Faverge committed
138
morse_parsec_data_of(parsec_data_collection_t *data_collection, ...)
Mathieu Faverge's avatar
Mathieu Faverge committed
139
{
Mathieu Faverge's avatar
Mathieu Faverge committed
140
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
141 142 143 144 145
    MORSE_desc_t *mdesc = pdesc->desc;
    va_list ap;
    int m, n;

    /* Get coordinates */
Mathieu Faverge's avatar
Mathieu Faverge committed
146
    va_start(ap, data_collection);
Mathieu Faverge's avatar
Mathieu Faverge committed
147 148 149 150 151 152 153 154 155 156
    m = va_arg(ap, unsigned int);
    n = va_arg(ap, unsigned int);
    va_end(ap);

    /* Offset by (i,j) to translate (m,n) in the global matrix */
    m += mdesc->i / mdesc->mb;
    n += mdesc->j / mdesc->nb;

#if defined(CHAMELEON_USE_MPI)
    /* TODO: change displacement in data_map when in distributed */
Mathieu Faverge's avatar
Mathieu Faverge committed
157
    //assert( data_collection->nodes == 1 );
Mathieu Faverge's avatar
Mathieu Faverge committed
158
#endif
Mathieu Faverge's avatar
Mathieu Faverge committed
159
    return parsec_data_create( pdesc->data_map + n * mdesc->lmt + m, data_collection,
Mathieu Faverge's avatar
Mathieu Faverge committed
160 161 162
                               morse_parsec_data_key( data_collection, m, n ),
                               mdesc->get_blkaddr( mdesc, m, n ),
                               mdesc->bsiz * MORSE_Element_Size(mdesc->dtyp) );
Mathieu Faverge's avatar
Mathieu Faverge committed
163 164
}

Mathieu Faverge's avatar
Mathieu Faverge committed
165
static inline parsec_data_t*
Mathieu Faverge's avatar
Mathieu Faverge committed
166
morse_parsec_data_of_key(parsec_data_collection_t *data_collection, parsec_data_key_t key)
Mathieu Faverge's avatar
Mathieu Faverge committed
167
{
Mathieu Faverge's avatar
Mathieu Faverge committed
168
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
169 170
    MORSE_desc_t *mdesc = pdesc->desc;
    int m, n;
Mathieu Faverge's avatar
Mathieu Faverge committed
171
    morse_parsec_key_to_coordinates(data_collection, key, &m, &n);
Mathieu Faverge's avatar
Mathieu Faverge committed
172 173 174

#if defined(CHAMELEON_USE_MPI)
    /* TODO: change displacement in data_map when in distributed */
Mathieu Faverge's avatar
Mathieu Faverge committed
175
    //assert( data_collection->nodes == 1 );
Mathieu Faverge's avatar
Mathieu Faverge committed
176
#endif
Mathieu Faverge's avatar
Mathieu Faverge committed
177
    return parsec_data_create( pdesc->data_map + key, data_collection, key,
Mathieu Faverge's avatar
Mathieu Faverge committed
178 179
                               mdesc->get_blkaddr( mdesc, m, n ),
                               mdesc->bsiz * MORSE_Element_Size(mdesc->dtyp) );
Mathieu Faverge's avatar
Mathieu Faverge committed
180 181
}

Mathieu Faverge's avatar
Mathieu Faverge committed
182
#ifdef parsec_PROF_TRACE
Mathieu Faverge's avatar
Mathieu Faverge committed
183
static inline int
Mathieu Faverge's avatar
Mathieu Faverge committed
184
morse_parsec_key_to_string(parsec_data_collection_t *data_collection, parsec_data_key_t key, char * buffer, uint32_t buffer_size)
Mathieu Faverge's avatar
Mathieu Faverge committed
185
{
Mathieu Faverge's avatar
Mathieu Faverge committed
186
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
187 188
    MORSE_desc_t *mdesc = pdesc->desc;
    int m, n, res;
Mathieu Faverge's avatar
Mathieu Faverge committed
189
    morse_parsec_key_to_coordinates(data_collection, key, &m, &n);
Mathieu Faverge's avatar
Mathieu Faverge committed
190 191 192
    res = snprintf(buffer, buffer_size, "(%d, %d)", m, n);
    if (res < 0)
    {
Mathieu Faverge's avatar
Mathieu Faverge committed
193 194
        printf("error in key_to_string for tile (%u, %u) key: %u\n",
               (unsigned int)m, (unsigned int)n, datakey);
Mathieu Faverge's avatar
Mathieu Faverge committed
195 196 197 198 199
    }
    return res;
}
#endif

200
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
201
 *  Create data descriptor
202
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
203 204
void RUNTIME_desc_create( MORSE_desc_t *mdesc )
{
Mathieu Faverge's avatar
Mathieu Faverge committed
205
    parsec_data_collection_t *data_collection;
Mathieu Faverge's avatar
Mathieu Faverge committed
206 207 208 209
    morse_parsec_desc_t *pdesc;
    int comm_size;

    pdesc = malloc( sizeof(morse_parsec_desc_t) );
Mathieu Faverge's avatar
Mathieu Faverge committed
210
    data_collection = (parsec_data_collection_t*)pdesc;
Mathieu Faverge's avatar
Mathieu Faverge committed
211 212

    /* Super setup */
Mathieu Faverge's avatar
Mathieu Faverge committed
213
    comm_size = RUNTIME_comm_size( NULL );
Mathieu Faverge's avatar
Mathieu Faverge committed
214 215 216 217 218 219 220 221 222 223 224
    data_collection->nodes  = comm_size;
    data_collection->myrank = mdesc->myrank;

    data_collection->data_key    = morse_parsec_data_key;
    data_collection->rank_of     = morse_parsec_rank_of;
    data_collection->rank_of_key = morse_parsec_rank_of_key;
    data_collection->data_of     = morse_parsec_data_of;
    data_collection->data_of_key = morse_parsec_data_of_key;
    data_collection->vpid_of     = morse_parsec_vpid_of;
    data_collection->vpid_of_key = morse_parsec_vpid_of_key;
#if defined(parsec_PROF_TRACE)
Mathieu Faverge's avatar
Mathieu Faverge committed
225 226
    {
        int rc;
Mathieu Faverge's avatar
Mathieu Faverge committed
227 228 229
        data_collection->key_to_string = morse_parsec_key_to_string;
        data_collection->key           = NULL;
        rc = asprintf(&(data_collection->key_dim), "(%d, %d)", mdesc->lmt, mdesc->lnt);
Mathieu Faverge's avatar
Mathieu Faverge committed
230 231 232
        (void)rc;
    }
#endif
Mathieu Faverge's avatar
Mathieu Faverge committed
233
    data_collection->memory_registration_status = MEMORY_STATUS_UNREGISTERED;
Mathieu Faverge's avatar
Mathieu Faverge committed
234

Mathieu Faverge's avatar
Mathieu Faverge committed
235
    pdesc->data_map = calloc( mdesc->lmt * mdesc->lnt, sizeof(parsec_data_t*) );
Mathieu Faverge's avatar
Mathieu Faverge committed
236 237 238 239 240

    /* Double linking */
    pdesc->desc     = mdesc;
    mdesc->schedopt = pdesc;

Mathieu Faverge's avatar
Mathieu Faverge committed
241 242
    parsec_dtd_data_collection_init(data_collection);

243
    /* arena init */
Mathieu Faverge's avatar
Mathieu Faverge committed
244 245
    pdesc->arena_index = 0;

246 247
    /* taskpool init to bypass a requirement of PaRSEC  */
#if defined(CHAMELEON_USE_MPI)
248 249 250 251 252
    /* Look if an arena already exists for this descriptor */
    {
        morse_parsec_arena_t *arena = morse_parsec_registered_arenas;
        size_t size = mdesc->mb * mdesc->nb * MORSE_Element_Size(mdesc->dtyp);
        int i;
253

254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
        for(i=0; i<morse_parsec_nb_arenas; i++, arena++) {
            if ( size == arena->size) {
                pdesc->arena_index = i;
                break;
            }
        }

        if (i == morse_parsec_nb_arenas) {
            parsec_datatype_t datatype;

            /* Create a taskpool to make sur the system is initialized */
            if ( i == 0 ) {
                parsec_taskpool_t *tp = parsec_dtd_taskpool_new();
                parsec_taskpool_free( tp );
            }

            /* Internal limitation of PaRSEC */
            assert(morse_parsec_nb_arenas < MORSE_PARSEC_DTD_NO_OF_ARENA);

            switch(mdesc->dtyp) {
            case MorseInteger:       datatype = parsec_datatype_int32_t; break;
            case MorseRealFloat:     datatype = parsec_datatype_float_t; break;
            case MorseRealDouble:    datatype = parsec_datatype_double_t; break;
            case MorseComplexFloat:  datatype = parsec_datatype_complex_t; break;
            case MorseComplexDouble: datatype = parsec_datatype_double_complex_t; break;
            default: morse_fatal_error("MORSE_Element_Size", "undefined type"); break;
            }

            /* Register the new arena */
Mathieu Faverge's avatar
Mathieu Faverge committed
283 284
            parsec_matrix_add2arena( parsec_dtd_arenas[i], datatype, matrix_UpperLower, 1,
                                     mdesc->mb, mdesc->nb, mdesc->mb, PARSEC_ARENA_ALIGNMENT_SSE, -1 );
285 286 287 288 289
            arena->size = size;
            pdesc->arena_index = i;
            morse_parsec_nb_arenas++;
        }
    }
290
#endif
Mathieu Faverge's avatar
Mathieu Faverge committed
291 292 293 294 295 296
    /* /\* Overwrite the leading dimensions to store the padding *\/ */
    /* mdesc->llm = mdesc->mb * mdesc->lmt; */
    /* mdesc->lln = mdesc->nb * mdesc->lnt; */
    return;
}

297
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
298
 *  Destroy data descriptor
299
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
300 301 302
void RUNTIME_desc_destroy( MORSE_desc_t *mdesc )
{
    morse_parsec_desc_t *pdesc = (morse_parsec_desc_t*)(mdesc->schedopt);
303 304 305
    if ( pdesc == NULL ) {
        return;
    }
Mathieu Faverge's avatar
Mathieu Faverge committed
306 307

    if ( pdesc->data_map != NULL ) {
Mathieu Faverge's avatar
Mathieu Faverge committed
308
        parsec_data_t **data = pdesc->data_map;
Mathieu Faverge's avatar
Mathieu Faverge committed
309 310 311 312 313
        int nb_local_tiles = mdesc->lmt * mdesc->lnt;
        int i;

        for(i=0; i<nb_local_tiles; i++, data++)
        {
Mathieu Faverge's avatar
Mathieu Faverge committed
314 315 316
            if (*data) {
                parsec_data_destroy( *data );
            }
Mathieu Faverge's avatar
Mathieu Faverge committed
317 318 319 320 321
        }

        free( pdesc->data_map );
        pdesc->data_map = NULL;
    }
Mathieu Faverge's avatar
Mathieu Faverge committed
322 323 324

    parsec_dtd_data_collection_fini( (parsec_data_collection_t *)pdesc );

Mathieu Faverge's avatar
Mathieu Faverge committed
325
    free(pdesc);
326
    mdesc->schedopt = NULL;
Mathieu Faverge's avatar
Mathieu Faverge committed
327 328 329
    return;
}

330
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
331
 *  Acquire data
332
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
333
int RUNTIME_desc_acquire( const MORSE_desc_t *desc )
Mathieu Faverge's avatar
Mathieu Faverge committed
334 335
{
    (void)desc;
Mathieu Faverge's avatar
Mathieu Faverge committed
336
    return MORSE_SUCCESS;
Mathieu Faverge's avatar
Mathieu Faverge committed
337 338
}

339
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
340
 *  Release data
341
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
342
int RUNTIME_desc_release( const MORSE_desc_t *desc )
Mathieu Faverge's avatar
Mathieu Faverge committed
343 344 345 346 347
{
    (void)desc;
    return MORSE_SUCCESS;
}

348
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
349
 *  Flush cached data
350
 */
Mathieu Faverge's avatar
Mathieu Faverge committed
351
void RUNTIME_flush()
Mathieu Faverge's avatar
Mathieu Faverge committed
352 353 354
{
}

Mathieu Faverge's avatar
Mathieu Faverge committed
355 356
void RUNTIME_desc_flush( const MORSE_desc_t     *desc,
                         const MORSE_sequence_t *sequence )
Mathieu Faverge's avatar
Mathieu Faverge committed
357
{
Mathieu Faverge's avatar
Mathieu Faverge committed
358 359 360
    parsec_taskpool_t* PARSEC_dtd_taskpool = (parsec_taskpool_t *)(sequence->schedopt);

    parsec_dtd_data_flush_all( PARSEC_dtd_taskpool, (parsec_data_collection_t*)(desc->schedopt) );
Mathieu Faverge's avatar
Mathieu Faverge committed
361
}
Mathieu Faverge's avatar
Mathieu Faverge committed
362

Mathieu Faverge's avatar
Mathieu Faverge committed
363 364 365
void RUNTIME_data_flush( const MORSE_sequence_t *sequence,
                         const MORSE_desc_t *A, int Am, int An )
{
366 367 368 369 370 371 372 373 374 375
    /*
     * For now, we do nothing in this function as in PaRSEC, once the data is
     * flushed it cannot be reused in the same sequence, when this issue will be
     * fixed, we will uncomment this function
     */
    /* parsec_taskpool_t* PARSEC_dtd_taskpool = (parsec_taskpool_t *)(sequence->schedopt); */
    /* parsec_dtd_data_flush( PARSEC_dtd_taskpool, RTBLKADDR( A, MORSE_Complex64_t, Am, An ) ); */

    (void)sequence; (void)A; (void)Am; (void)An;
    return;
Mathieu Faverge's avatar
Mathieu Faverge committed
376 377
}

378 379 380 381 382 383 384 385
#if defined(CHAMELEON_USE_MIGRATE)
void RUNTIME_data_migrate( const MORSE_sequence_t *sequence,
                           const MORSE_desc_t *A, int Am, int An, int new_rank )
{
    (void)sequence; (void)A; (void)Am; (void)An; (void)new_rank;
}
#endif

386
/**
Mathieu Faverge's avatar
Mathieu Faverge committed
387
 *  Get data addr
388
 */
389
void *RUNTIME_desc_getaddr( const MORSE_desc_t *desc, int m, int n )
Mathieu Faverge's avatar
Mathieu Faverge committed
390 391 392 393
{
    assert(0); /* This should not be called because we also need the handle to match the address we need. */
    return desc->get_blkaddr( desc, m, n );
}