.gitlab-ci.yml 15.1 KB
Newer Older
1 2

before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
3
  - make demo
4 5
  - make data
  - make germline
6
  - make -C browser sha1
7
  - cp -p doc/analysis-example.vidjil browser/
8 9

stages:
10
  - test_tools
11
  - test_quality
12
  - test_unit
13
  - deploy_review
14
  - test_germlines
15
  - test_functional
flothoni's avatar
flothoni committed
16
  - test_tutorial
17
  - test_shouldvdj
18
  - test_functional_external
19
  - coverage
20
  - prepare_release
21
  - tgz_release
Mathieu Giraud's avatar
Mathieu Giraud committed
22 23
  - valgrind_unit
  - valgrind_functional
24
  - multiple_tests
25
  - benchmark
26
  - publish_release
27
  - deploy_prod
28

29 30
# Anchors

31 32
.coverage_dependency:
  artifacts: &lcov_artifact
33 34
    paths:
      - algo/lcov_test_*.filtered
35
    expire_in: 1 week
36

37 38 39 40 41 42
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

43 44 45 46 47 48 49

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
50
    - make -C germline tests
51
  only:
52
    - /^feature-.*g.*\/.*$/
53

54

Mathieu Giraud's avatar
Mathieu Giraud committed
55
# Includes
56 57

include:
Mathieu Giraud's avatar
Mathieu Giraud committed
58
  - local: '/doc/.gitlab-ci.yml' # Doc
59
  - local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
60
  - local: 'algo/.gitlab-ci.yml' # Vidjil-algo pipelines
Mathieu Giraud's avatar
Mathieu Giraud committed
61
  - template: Code-Quality.gitlab-ci.yml # Code quality
62

Mathieu Giraud's avatar
Mathieu Giraud committed
63
# Algorithm
64 65 66

test_algo_unit:
  stage: test_unit
67
  script: make -C algo unit_coverage
68 69
  artifacts:
    <<: *lcov_artifact
70
  only:
71
    - /^feature-.*a.*\/.*$/
72
    - schedules
73

74 75
test_algo_should:
  stage: test_functional
76
  script: make -C algo should_coverage
77
  artifacts:
78
    <<: *lcov_artifact
79 80
    reports:
      junit: algo/tests/should.xml
81
  only:
82
    - /^feature-.*a.*\/.*$/
83
    - schedules
84 85 86

test_algo_shouldvdj:
  stage: test_shouldvdj
87
  script: make -C algo shouldvdj_coverage
88 89
  artifacts:
    <<: *lcov_artifact
90 91
  only:
    - /^feature-.*a.*\/.*$/
92 93
    - schedules
  
94 95
algo_coverage:
  stage: coverage
96
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
97 98 99
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
100 101
  artifacts:
    paths:
102
      - coverage/
103
    expire_in: 1 week
104
  only:
105
    - /^feature-.*a.*\/.*$/
106
    - schedules
107

Mathieu Giraud's avatar
Mathieu Giraud committed
108 109
algo_valgrind_unit:
  stage: valgrind_unit
110
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
111
  only:
112
    - /^feature-.*a.*\/.*$/
113
    - schedules
114 115
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
116

117
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
118
  stage: valgrind_functional
119
  script: make -C algo valgrind_should
120 121 122 123 124
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
125 126
  when: manual
  only:
127
    - /^feature-.*a.*\/.*$/
128
    - schedules
129 130 131

release_valgrind_functional:
  <<: *valgrind_functional
132
  allow_failure: true
133 134
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
135

136 137 138 139 140
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
141
    - /^feature-.*a.*\/.*$/
142

143 144
tgz_release:
  stage: tgz_release
145 146
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  only:
147
    - /^feature-.*a.*\/release$/
148
  artifacts:
149
    paths:
150 151
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
152
    expire_in: 10 years
153
  allow_failure: false
154 155
  tags:
    - ubuntu-16.04-amd64
156

157
copy_release:
158
  stage: publish_release
159 160 161 162 163 164 165 166
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
167
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
168
  when: manual
169 170 171
  only:
    - /^feature-.*a.*\/release$/

172 173
deploy_release_prod:
  stage: deploy_prod
174
  when: manual
175 176 177 178 179
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
180
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
181
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
182 183
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
184

Mathieu Giraud's avatar
Mathieu Giraud committed
185
# Client
186

187
test_browser_unit:
188
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
189
  script: make unit_browser
190 191 192
  artifacts:
    paths:
    - browser/
193
    when: always
194
  only:
195 196
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
197
    - /^prod-client/
198
    - dev
199
    - schedules
200 201
  tags:
    - web
202

203
.browser-functional:
204
  stage: test_functional
205
  retry: 2
206 207 208
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
209
    - rvm use 2.6.1
210
    - HEADLESS=1 make -C browser/test functional
211 212
  artifacts:
    paths:
213 214
    - browser/ 
    when: always
215 216
    reports:
      junit: browser/test/test/TEST-*.xml
217
  only:
218 219
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
220
    - /^prod-client/
221
    - schedules
222 223
  tags:
    - web
224

225 226 227 228 229
.browser-functional-chrome:
  extends: .browser-functional
  variables:
    WATIR_CHROME: "1"

230
.browser-functional-external:
231
  stage: test_functional_external
232
  retry: 2
233 234 235
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
236
    - rvm use 2.6.1
237
    - HEADLESS=1 make -C browser/test external
238 239 240
  artifacts:
    paths:
    - browser/
241
    when: always
242 243
    reports:
      junit: browser/test/test/TEST-*.xml
244
  only:
245 246
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
247
    - /^prod-client/
248
    - schedules
249 250
  tags:
    - web
251

252 253 254 255 256
.browser-functional-external-chrome:
  extends: .browser-functional-external
  variables:
    WATIR_CHROME: "1"
    
257
ff32-browser-functional:
258 259
  extends: .browser-functional
  variables:
260
    FIREFOX_VERSION: 32
261

262
ff45-browser-functional:
263 264
  extends: .browser-functional
  variables:
265 266
    FIREFOX_VERSION: 45

267
chrome-browser-functional:
268 269
  extends: .browser-functional-chrome
        
270
old-chrome-browser-functional:
271
  extends: .browser-functional-chrome
272 273 274
  tags:
    - legacy

275
ff32-browser-functional-external:
276 277
  extends: .browser-functional-external
  variables:
278
    FIREFOX_VERSION: 32
279

280
ff45-browser-functional-external:
281 282
  extends: .browser-functional-external
  variables:
283
    FIREFOX_VERSION: 45
284

285
chrome-browser-functional-external:
286 287
  extends: .browser-functional-external-chrome
    
288
old-chrome-browser-functional-external-chrome:
289 290 291
  extends: .browser-functional-external-chrome
  tags:
    - legacy
292

flothoni's avatar
flothoni committed
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310


.browser-tutorial:
  stage: test_tutorial
  retry: 2
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.6.1
    - HEADLESS=1 make test_tutorial
  artifacts:
    paths:
    - browser/
    reports:
      junit: browser/test/test/TEST-*.xml
  only:
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
311
    - /^prod-client/
flothoni's avatar
flothoni committed
312
    - schedules
313
    - /tutorial/
flothoni's avatar
flothoni committed
314
  tags:
315
    - web-pandoc
flothoni's avatar
flothoni committed
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340

.browser-tutorial-chrome:
  extends: .browser-tutorial
  variables:
    WATIR_CHROME: "1"


ff32-browser-tutorial:
  extends: .browser-tutorial
  variables:
    FIREFOX_VERSION: 32

ff45-browser-tutorial:
  extends: .browser-tutorial
  variables:
    FIREFOX_VERSION: 45

chrome-browser-tutorial:
  extends: .browser-tutorial-chrome
        
old-chrome-browser-tutorial:
  extends: .browser-tutorial-chrome
  tags:
    - legacy

Mathieu Giraud's avatar
Mathieu Giraud committed
341 342
# Code quality

343
code_quality:
Mathieu Giraud's avatar
Mathieu Giraud committed
344 345 346 347 348
  stage: test_quality
  artifacts:
    paths: [gl-code-quality-report.json]

code_quality_custom:
349
  stage: test_quality
350
  script: make -C browser quality
351
  only:
352 353
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
354
    - /^prod-client/
355
    - dev
356
    - schedules
357 358
  tags:
    - web
359

Ryan Herbert's avatar
Ryan Herbert committed
360 361 362 363 364
# Server

test_server_unit:
    stage: test_unit
    script:
365 366
      - virtualenv $CI_COMMIT_SHORT_SHA
      - source $CI_COMMIT_SHORT_SHA/bin/activate
367
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
368
      - make -C server install_web2py_standalone
369
      - make -C server launch_fuse_server
370
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
371
      - make -C server kill_fuse_server
372
      - deactivate
373 374 375
    artifacts:
      reports:
        junit: server/web2py/test-reports/TEST*xml
Ryan Herbert's avatar
Ryan Herbert committed
376
    only:
377 378
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
379
      - /^prod-server/
Ryan Herbert's avatar
Ryan Herbert committed
380
      - dev
381
      - schedules
Ryan Herbert's avatar
Ryan Herbert committed
382 383 384
    tags:
      - web

385
.test_server_functional:
386 387
  stage: test_functional
  script:
388 389
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
390
    - sed -i '/\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
391 392 393
    - sed -i '/\/var\/lib\/mysql/d' ./docker/docker-compose.yml
    - sed -i 's/\/opt\/vidjil\/databases/databases/' ./docker/docker-compose.yml
    - echo -e "volumes:\n    databases:" >> ./docker/docker-compose.yml
394
    - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml
395
    - cd docker/vidjil-server/conf/ && mv defs.py defs_https.py && mv defs_http.py defs.py && cd ../../..
396
    - make germline && cp browser/js/germline.js docker/vidjil-client/conf
397
    - mkdir -p docker/postfix/ssl
398
    - cd docker && docker-compose up -d && cd ..
399
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_tests.sh
400
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
401
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
402
    - source /etc/profile.d/rvm.sh
403
    - rvm use 2.6.1
404 405
    - HEADLESS=1 make functional_server
  after_script:
406
    - cd docker && docker-compose down -v
407
    - docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test
408 409 410
  artifacts:
    reports:
      junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml
411 412 413
    when: always
    expire_in: 1 week
    paths:
Thonier Florian's avatar
Thonier Florian committed
414
    - server/web2py/ap‎plications/vidjil‎/tests/
415 416 417
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
418
    - /^prod-server/
419
    - dev
420
    - schedules
421
  tags:
422
    - x86_64
423
    - docker
424

425
ff32-server-functional:
426 427 428 429
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 32

430
ff45-server-functional:
431 432 433 434 435
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 45


436 437 438

# Benchmark

439
.benchmark-algo:
440
  image:  gcc:6.3
441
  extends: .install-algo-dependencies
442 443
  stage: benchmark
  script:
444
    - cd algo/tests ; python3 benchmark-releases.py -r 3 -bIc
445 446 447
  artifacts:
    paths:
      - algo/tests/benchmark.log
448 449 450 451
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
  tags:
452
    - cidocker
453

454 455 456 457 458 459 460 461 462 463
benchmark-algo:
  extends: .benchmark-algo
  script:
    - cd algo/tests ; python3 benchmark-releases.py -r 3 -b -c -i 2018.10 -i 2019.03 -i 2020.04 -i 2021.02.2 -i alpha-2102

benchmark-algo-full:
  extends: .benchmark-algo
  script:
    - cd algo/tests ; python3 benchmark-releases.py -r 3 -bIc

464

Ryan Herbert's avatar
Ryan Herbert committed
465 466
# Deployment

467 468
deploy_review:
  stage: deploy_review
469
  script:
470
    - make -C browser icons
471
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
472
    - rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG
473
  environment:
474
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
475
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
476
    on_stop: stop_deploy_review
477
  only:
478 479
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
480 481
  tags:
    - web
482

483 484 485
deploy_review_server:
  stage: deploy_review
  script:
486 487
    - make -C browser icons
    - cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
488
    - sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
489
    - 'sed -i "s/\"server_id\" : \"\"/\"server_id\" : \"${CI_BUILD_REF_SLUG}.server.ci.vidjil.org\"/" ./browser/js/conf.js'
490
    - sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
491
    - sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py
492
    - sed -ri 's/^(uwsgi_pass )\$upstream_(.*)$/\1\2/g' docker/vidjil-client/conf/uwsgi.conf
493 494
    - mkdir browser/cgi
    - cp docker/vidjil-client/conf/*.cgi browser/cgi
495 496
    - mv docker docker_${CI_BUILD_REF_SLUG}
    - cd docker_${CI_BUILD_REF_SLUG}
497 498
    - TMP_DOCKER=$(tempfile)
    - cp docker-compose.yml $TMP_DOCKER
499 500
    - awk -f ci/for-review.awk $TMP_DOCKER > docker-compose.yml
    - rm -f $TMP_DOCKER
501
    - sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml
502
    - cd ..
503
    - rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac)
504
    - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers; sleep 20s; docker-compose exec -T uwsgi bash -c 'cd /usr/share/vidjil/server/web2py/applications/vidjil/scripts; /usr/local/bin/gosu 1005 bash launch_python_script.sh load-sql.py -n /usr/share/vidjil/docker/ci/ci.sql'"
505 506
  environment:
    name: review_server/$CI_BUILD_REF_NAME
507
    url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/
508
    on_stop: stop_deploy_review_server
509
    auto_stop_in: 2 months
510 511 512 513
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/

514
deploy_docker:
515
  stage: deploy_prod
516
  script:
517
    - export CUR_DATE=`date +%Y-%m-%d`
518
    - export TYPE=`echo $CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'`
519 520
    - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
    - export TAG="vidjil/$TYPE:$CUR_DATE-$SHA"
521
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
522
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE
523 524 525 526
    - docker push $TAG
    - docker tag $TAG "vidjil/$TYPE:latest"
    - docker push "vidjil/$TYPE:latest"
    - docker rmi $TAG "vidjil/$TYPE:latest"
527 528
  when: manual
  only:
529 530
    - /^prod-server/
    - /^prod-client/
531 532
  tags:
    - web
533
    - docker
534

535
deploy_prod:
536
  stage: deploy_prod
537
  script:
538
    - ssh $PROD_CLIENT_SERVER "
539
         cd $PROD_CLIENT_PATH;"
540 541
  environment:
    name: production
542
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
543
  only:
544
    - /^prod-client/
545 546
  tags:
    - web
547

548 549 550 551 552
deploy_germlines:
  stage: deploy_prod
  script:
    - make -C germline get-all-data
    - make -C germline js
553
    - date=$(date +%Y-%m-%d)
554 555 556 557 558 559 560 561
    - tar cvzf germline-$date.tar.gz germline/*/*.fa germline/IMGT_RELEASE browser/js/germline.js
    - scp germline-$date.tar.gz $VIDJIL_WWW:$REMOTE_GERMLINE_DIR
  only:
    - manual
    - /^feature-.*g.*\/.*$/
    


562 563
stop_deploy_review:
  stage: deploy_review
564 565 566
  variables:
    GIT_STRATEGY: none
  script:
567
    - ssh $REVIEW_SERVER "rm -rf $REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG"
568 569
  when: manual
  environment:
570
    name: review/$CI_BUILD_REF_NAME
571
    action: stop
572
  only:
573 574
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
575
  tags:
576
    - web
577 578 579 580

stop_deploy_review_server:
  stage: deploy_review
  script:
581
    - ssh $REVIEW_SERVER "set -o xtrace; docker-compose -f $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG/docker_$CI_BUILD_REF_SLUG/docker-compose.yml down -v ; rm -rf $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG; $REVIEW_SERVER_PATH_SERVER/uninstall_certs.sh $CI_BUILD_REF_SLUG"
582 583 584 585 586 587 588
  when: manual
  environment:
    name: review_server/$CI_BUILD_REF_NAME
    action: stop
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/