pzunmqrrh.c 18.6 KB
Newer Older
1 2
/**
 *
3 4
 * @copyright (c) 2009-2014 The University of Tennessee and The University
 *                          of Tennessee Research Foundation.
5
 *                          All rights reserved.
6
 * @copyright (c) 2012-2016 Inria. All rights reserved.
7
 * @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
 *
 **/

/**
 *
 * @file pzunmqrrh.c
 *
 *  MORSE auxiliary routines
 *  MORSE is a software package provided by Univ. of Tennessee,
 *  Univ. of California Berkeley and Univ. of Colorado Denver
 *
 * @version 2.5.0
 * @comment This file has been automatically generated
 *          from Plasma 2.5.0 for MORSE 1.0.0
 * @author Hatem Ltaief
 * @author Jakub Kurzak
 * @author Dulceneia Becker
 * @author Mathieu Faverge
 * @author Emmanuel Agullo
 * @author Cedric Castagnede
 * @date 2010-11-15
 * @precisions normal z -> s d c
 *
 **/
32
#include "control/common.h"
33 34 35 36 37

#define A(m,n) A,  (m),  (n)
#define B(m,n) B,  (m),  (n)
#define T(m,n) T,  (m),  (n)
#define T2(m,n) T,  (m),  ((n)+A->nt)
38
#if defined(CHAMELEON_COPY_DIAG)
39
#define D(m,n) D, ((m)/BS), 0
40
#else
41
#define D(m,n) A, (m), (n)
42
#endif
43

Mathieu Faverge's avatar
Mathieu Faverge committed
44
/*******************************************************************************
45 46 47
 *  Parallel application of Q using tile V - QR factorization (reduction
 *  Householder) - dynamic scheduling
 **/
Mathieu Faverge's avatar
Mathieu Faverge committed
48 49 50
void morse_pzunmqrrh( MORSE_enum side, MORSE_enum trans,
                      MORSE_desc_t *A, MORSE_desc_t *B, MORSE_desc_t *T, MORSE_desc_t *D, int BS,
                      MORSE_sequence_t *sequence, MORSE_request_t *request )
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
{
    MORSE_context_t *morse;
    MORSE_option_t options;
    size_t ws_worker = 0;
    size_t ws_host = 0;

    int k, m, n;
    int K, M, RD, lastRD;
    int ldaM, ldam, ldan, ldaMRD;
    int ldbM, ldbm, ldbMRD;
    int tempMm, tempkn, tempnn, tempmm, tempMRDm, tempkmin;
    int ib;

    morse = morse_context_self();
    if (sequence->status != MORSE_SUCCESS)
        return;
    RUNTIME_options_init(&options, morse, sequence, request);

    ib = MORSE_IB;

    /*
     * zunmqr = A->nb * ib
     * ztsmqr = A->nb * ib
     * zttmqr = A->nb * ib
     */
    ws_worker = A->nb * ib;

Mathieu Faverge's avatar
Mathieu Faverge committed
78
#if defined(CHAMELEON_USE_CUDA)
79 80 81 82 83
    /* Worker space
     *
     * zunmqr = A->nb * ib
     * ztsmqr = 2 * A->nb * ib
     */
84
    ws_worker = chameleon_max( ws_worker, ib * A->nb * 2 );
85 86 87 88 89 90 91
#endif

    ws_worker *= sizeof(MORSE_Complex64_t);
    ws_host   *= sizeof(MORSE_Complex64_t);

    RUNTIME_options_ws_alloc( &options, ws_worker, ws_host );

92
    K = chameleon_min(A->mt, A->nt);
93 94 95 96 97 98
    if (side == MorseLeft ) {
        if (trans == MorseConjTrans) {
            /*
             *  MorseLeft / MorseConjTrans
             */
            for (k = 0; k < K; k++) {
99
                RUNTIME_iteration_push(morse, k);
100

101 102 103
                tempkn = k == A->nt-1 ? A->n-k*A->nb : A->nb;
                for (M = k; M < A->mt; M += BS) {
                    tempMm   = M == A->mt-1 ? A->m-M*A->mb : A->mb;
104
                    tempkmin = chameleon_min(tempMm, tempkn);
105 106
                    ldaM = BLKLDD(A, M);
                    ldbM = BLKLDD(B, M);
107
#if defined(CHAMELEON_COPY_DIAG)
108 109 110 111
                    MORSE_TASK_zlacpy(
                        &options,
                        MorseLower, tempMm, tempkmin, A->nb,
                        A(M, k), ldaM,
112
                        D(M, k), ldaM );
Mathieu Faverge's avatar
Mathieu Faverge committed
113
#if defined(CHAMELEON_USE_CUDA)
114 115 116 117
                    MORSE_TASK_zlaset(
                        &options,
                        MorseUpper, tempMm, tempkmin,
                        0., 1.,
118
                        D(M, k), ldaM );
119
#endif
120 121 122 123 124 125
#endif
                    for (n = 0; n < B->nt; n++) {
                        tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
                        MORSE_TASK_zunmqr(
                            &options,
                            side, trans,
BOUCHERIE Raphael's avatar
minor  
BOUCHERIE Raphael committed
126
                            tempMm, tempnn, tempkmin, ib, T->nb,
127
                            D(M, k), ldaM,
128 129 130
                            T(M, k), T->mb,
                            B(M, n), ldbM);
                    }
131
                    for (m = M+1; m < chameleon_min(M+BS, A->mt); m++) {
132 133 134 135 136
                        tempmm = m == A->mt-1 ? A->m-m*A->mb : A->mb;
                        ldbm = BLKLDD(B, m);
                        ldam = BLKLDD(A, m);
                        for (n = 0; n < B->nt; n++) {
                            tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
137 138 139 140 141 142 143 144 145

                            RUNTIME_data_migrate( sequence, B(M, n),
                                                  B->get_rankof( B, m, n ) );
                            RUNTIME_data_migrate( sequence, B(m, n),
                                                  B->get_rankof( B, m, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempnn, tempkn, 0, ib, T->nb,
146
                                A(m, k), ldam,
147 148 149
                                T(m, k), T->mb,
                                B(M, n), ldbM,
                                B(m, n), ldbm);
150 151 152 153 154 155 156 157 158 159 160
                        }
                    }
                }
                for (RD = BS; RD < A->mt-k; RD *= 2) {
                    for (M = k; M+RD < A->mt; M += 2*RD) {
                        tempMRDm = M+RD == A->mt-1 ? A->m-(M+RD)*A->mb : A->mb;
                        ldbM   = BLKLDD(B, M   );
                        ldbMRD = BLKLDD(B, M+RD);
                        ldaMRD = BLKLDD(A, M+RD);
                        for (n = 0; n < B->nt; n++) {
                            tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
161 162 163 164 165 166 167 168 169

                            RUNTIME_data_migrate( sequence, B(M, n),
                                                  B->get_rankof( B, M+RD, n ) );
                            RUNTIME_data_migrate( sequence, B(M+RD, n),
                                                  B->get_rankof( B, M+RD, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempMRDm, tempnn, tempkn, tempMRDm, ib, T->nb,
170
                                A (M+RD, k), ldaMRD,
171 172 173
                                T2(M+RD, k), T->mb,
                                B (M,    n), ldbM,
                                B (M+RD, n), ldbMRD);
174 175 176
                        }
                    }
                }
177

178 179 180 181 182 183
                /* Restore the original location of the tiles */
                for (n = 0; n < B->nt; n++) {
                    RUNTIME_data_migrate( sequence, B(k, n),
                                          B->get_rankof( B, k, n ) );
                }

184
                RUNTIME_iteration_pop(morse);
185 186 187 188 189 190
            }
        } else {
            /*
             *  MorseLeft / MorseNoTrans
             */
            for (k = K-1; k >= 0; k--) {
191
                RUNTIME_iteration_push(morse, k);
192

193 194 195 196 197 198 199 200 201 202 203 204
                tempkn = k == A->nt-1 ? A->n-k*A->nb : A->nb;
                lastRD = 0;
                for (RD = BS; RD < A->mt-k; RD *= 2)
                    lastRD = RD;
                for (RD = lastRD; RD >= BS; RD /= 2) {
                    for (M = k; M+RD < A->mt; M += 2*RD) {
                        tempMRDm = M+RD == A->mt-1 ? A->m-(M+RD)*A->mb : A->mb;
                        ldbM   = BLKLDD(B, M   );
                        ldbMRD = BLKLDD(B, M+RD);
                        ldaMRD = BLKLDD(A, M+RD);
                        for (n = 0; n < B->nt; n++) {
                            tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
205 206 207 208 209 210 211 212 213

                            RUNTIME_data_migrate( sequence, B(M, n),
                                                  B->get_rankof( B, M+RD, n ) );
                            RUNTIME_data_migrate( sequence, B(M+RD, n),
                                                  B->get_rankof( B, M+RD, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempMRDm, tempnn, tempkn, tempMRDm, ib, T->nb,
214
                                A (M+RD, k), ldaMRD,
215 216 217
                                T2(M+RD, k), T->mb,
                                B (M,    n), ldbM,
                                B (M+RD, n), ldbMRD);
218 219 220 221 222
                        }
                    }
                }
                for (M = k; M < A->mt; M += BS) {
                    tempMm   = M == A->mt-1 ? A->m-M*A->mb : A->mb;
223
                    tempkmin = chameleon_min(tempMm, tempkn);
224 225
                    ldaM = BLKLDD(A, M);
                    ldbM = BLKLDD(B, M);
226
                    for (m = chameleon_min(M+BS, A->mt)-1; m > M; m--) {
227 228 229 230 231
                        tempmm = m == A->mt-1 ? A->m-m*A->mb : A->mb;
                        ldbm = BLKLDD(B, m);
                        ldam = BLKLDD(A, m);
                        for (n = 0; n < B->nt; n++) {
                            tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
232 233 234 235 236 237 238 239 240

                            RUNTIME_data_migrate( sequence, B(M, n),
                                                  B->get_rankof( B, m, n ) );
                            RUNTIME_data_migrate( sequence, B(m, n),
                                                  B->get_rankof( B, m, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempnn, tempkn, 0, ib, T->nb,
241
                                A(m, k), ldam,
242 243 244
                                T(m, k), T->mb,
                                B(M, n), ldbM,
                                B(m, n), ldbm);
245 246
                        }
                    }
247
#if defined(CHAMELEON_COPY_DIAG)
248 249 250 251
                    MORSE_TASK_zlacpy(
                        &options,
                        MorseLower, tempMm, tempkmin, A->nb,
                        A(M, k), ldaM,
252
                        D(M, k), ldaM );
Mathieu Faverge's avatar
Mathieu Faverge committed
253
#if defined(CHAMELEON_USE_CUDA)
254 255 256 257
                    MORSE_TASK_zlaset(
                        &options,
                        MorseUpper, tempMm, tempkmin,
                        0., 1.,
258
                        D(M, k), ldaM );
259
#endif
260 261 262
#endif
                    for (n = 0; n < B->nt; n++) {
                        tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
263 264 265 266

                        RUNTIME_data_migrate( sequence, B(M, n),
                                              B->get_rankof( B, M, n ) );

267
                        MORSE_TASK_zunmqr(
268 269
                            &options, side, trans,
                            tempMm, tempnn, tempkmin, ib, T->nb,
270
                            D(M, k), ldaM,
271 272 273 274
                            T(M, k), T->mb,
                            B(M, n), ldbM);
                    }
                }
275
                RUNTIME_iteration_pop(morse);
276 277
            }
        }
Mathieu Faverge's avatar
Minor  
Mathieu Faverge committed
278 279
    }
    else {
280 281 282 283 284
        if (trans == MorseConjTrans) {
            /*
             *  MorseRight / MorseConjTrans
             */
            for (k = K-1; k >= 0; k--) {
285
                RUNTIME_iteration_push(morse, k);
286

287 288 289 290 291 292 293 294 295 296 297
                tempkn = k == A->nt-1 ? A->n-k*A->nb : A->nb;
                lastRD = 0;
                for (RD = BS; RD < A->mt-k; RD *= 2)
                    lastRD = RD;
                for (RD = lastRD; RD >= BS; RD /= 2) {
                    for (M = k; M+RD < A->mt; M += 2*RD) {
                        tempMRDm = M+RD == A->mt-1 ? A->m-(M+RD)*A->mb : A->mb;
                        ldaMRD = BLKLDD(A, M+RD);
                        for (m = 0; m < B->mt; m++) {
                            ldbm   = BLKLDD(B, m);
                            tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
298 299 300 301 302 303 304 305 306

                            RUNTIME_data_migrate( sequence, B(m, M),
                                                  B->get_rankof( B, m, M+RD ) );
                            RUNTIME_data_migrate( sequence, B(m, M+RD),
                                                  B->get_rankof( B, m, M+RD ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempMRDm, tempkn, tempmm, ib, T->nb,
307
                                A (M+RD, k), ldaMRD,
308 309 310
                                T2(M+RD, k), T->mb,
                                B (m, M), ldbm,
                                B (m, M+RD), ldbm);
311 312 313 314 315
                        }
                    }
                }
                for (M = k; M < A->mt; M += BS) {
                    tempMm   = M == A->mt-1 ? A->m-M*A->mb : A->mb;
316
                    tempkmin = chameleon_min(tempMm, tempkn);
317 318
                    ldaM = BLKLDD(A, M);
                    ldbM = BLKLDD(B, M);
319
                    for (n = chameleon_min(M+BS, A->mt)-1; n > M; n--) {
320 321 322 323 324
                        ldan = BLKLDD(A, n);
                        tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
                        for (m = 0; m < B->mt; m++) {
                            ldbm = BLKLDD(B, m);
                            tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
325 326 327 328 329 330 331 332 333

                            RUNTIME_data_migrate( sequence, B(m, M),
                                                  B->get_rankof( B, m, n ) );
                            RUNTIME_data_migrate( sequence, B(m, m),
                                                  B->get_rankof( B, m, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempnn, tempkn, 0, ib, T->nb,
334
                                A(n, k), ldan,
335 336 337
                                T(n, k), T->mb,
                                B(m, M), ldbm,
                                B(m, n), ldbm);
338 339
                        }
                    }
340
#if defined(CHAMELEON_COPY_DIAG)
341 342 343 344
                    MORSE_TASK_zlacpy(
                        &options,
                        MorseLower, tempMm, tempkmin, A->nb,
                        A(M, k), ldaM,
345
                        D(M, k), ldaM );
Mathieu Faverge's avatar
Mathieu Faverge committed
346
#if defined(CHAMELEON_USE_CUDA)
347 348 349 350
                    MORSE_TASK_zlaset(
                        &options,
                        MorseUpper, tempMm, tempkmin,
                        0., 1.,
351
                        D(M, k), ldaM );
352
#endif
353 354 355 356
#endif
                    for (m = 0; m < B->mt; m++) {
                        ldbm = BLKLDD(B, m);
                        tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
357 358 359 360

                        RUNTIME_data_migrate( sequence, B(m, M),
                                              B->get_rankof( B, m, M ) );

361 362 363
                        MORSE_TASK_zunmqr(
                            &options,
                            side, trans,
BOUCHERIE Raphael's avatar
minor  
BOUCHERIE Raphael committed
364
                            tempmm, tempMm, tempkmin, ib, T->nb,
365
                            D(M, k), ldaM,
366 367 368 369
                            T(M, k), T->mb,
                            B(m, M), ldbm);
                    }
                }
370 371

                RUNTIME_iteration_pop(morse);
372 373 374 375 376 377
            }
        } else {
            /*
             *  MorseRight / MorseNoTrans
             */
            for (k = 0; k < K; k++) {
378
                RUNTIME_iteration_push(morse, k);
379

380 381 382
                tempkn = k == A->nt-1 ? A->n-k*A->nb : A->nb;
                for (M = k; M < A->mt; M += BS) {
                    tempMm   = M == A->mt-1 ? A->m-M*A->mb : A->mb;
383
                    tempkmin = chameleon_min(tempMm, tempkn);
384
                    ldaM = BLKLDD(A, M);
385
#if defined(CHAMELEON_COPY_DIAG)
386 387 388 389
                    MORSE_TASK_zlacpy(
                        &options,
                        MorseLower, tempMm, tempkmin, A->nb,
                        A(M, k), ldaM,
390
                        D(M, k), ldaM );
Mathieu Faverge's avatar
Mathieu Faverge committed
391
#if defined(CHAMELEON_USE_CUDA)
392 393 394 395
                    MORSE_TASK_zlaset(
                        &options,
                        MorseUpper, tempMm, tempkmin,
                        0., 1.,
396
                        D(M, k), ldaM );
397
#endif
398 399 400 401 402 403 404
#endif
                    for (m = 0; m < B->mt; m++) {
                        ldbm = BLKLDD(B, m);
                        tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
                        MORSE_TASK_zunmqr(
                            &options,
                            side, trans,
BOUCHERIE Raphael's avatar
minor  
BOUCHERIE Raphael committed
405
                            tempmm, tempMm, tempkmin, ib, T->nb,
406
                            D(M, k), ldaM,
407 408 409
                            T(M, k), T->mb,
                            B(m, M), ldbm);
                    }
410
                    for (n = M+1; n < chameleon_min(M+BS,  A->mt); n++) {
411 412 413 414 415
                        tempnn = n == B->nt-1 ? B->n-n*B->nb : B->nb;
                        ldan = BLKLDD(A, n);
                        for (m = 0; m < B->mt; m++) {
                            tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
                            ldbm = BLKLDD(B, m);
416 417 418 419 420 421 422 423 424

                            RUNTIME_data_migrate( sequence, B(m, M),
                                                  B->get_rankof( B, m, n ) );
                            RUNTIME_data_migrate( sequence, B(m, n),
                                                  B->get_rankof( B, m, n ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempnn, tempkn, 0, ib, T->nb,
425
                                A(n, k), ldan,
426 427 428
                                T(n, k), T->mb,
                                B(m, M), ldbm,
                                B(m, n), ldbm);
429 430 431 432 433 434 435 436 437 438
                        }
                    }
                }
                for (RD = BS; RD < A->mt-k; RD *= 2) {
                    for (M = k; M+RD < A->mt; M += 2*RD) {
                        tempMRDm = M+RD == A->mt-1 ? A->m-(M+RD)*A->mb : A->mb;
                        ldaMRD = BLKLDD(A, M+RD);
                        for (m = 0; m < B->mt; m++) {
                            tempmm = m == B->mt-1 ? B->m-m*B->mb : B->mb;
                            ldbm   = BLKLDD(B, m);
439 440 441 442 443 444 445 446 447

                            RUNTIME_data_migrate( sequence, B(m, M),
                                                  B->get_rankof( B, m, M+RD ) );
                            RUNTIME_data_migrate( sequence, B(m, M+RD),
                                                  B->get_rankof( B, m, M+RD ) );

                            MORSE_TASK_ztpmqrt(
                                &options, side, trans,
                                tempmm, tempMRDm, tempkn, tempmm, ib, T->nb,
448
                                A (M+RD, k), ldaMRD,
449 450 451
                                T2(M+RD, k), T->mb,
                                B (m, M   ), ldbm,
                                B (m, M+RD), ldbm);
452 453 454
                        }
                    }
                }
455

456 457 458 459 460 461
                /* Restore the original location of the tiles */
                for (m = 0; m < B->mt; m++) {
                    RUNTIME_data_migrate( sequence, B(m, k),
                                          B->get_rankof( B, m, k ) );
                }

462
                RUNTIME_iteration_pop(morse);
463 464 465
            }
        }
    }
Mathieu Faverge's avatar
Mathieu Faverge committed
466

467 468
    RUNTIME_options_ws_free(&options);
    RUNTIME_options_finalize(&options, morse);
Mathieu Faverge's avatar
Mathieu Faverge committed
469
    (void)D;
470
}