.gitlab-ci.yml 13.3 KB
Newer Older
1 2

before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
3
  - make demo
4 5
  - make data
  - make germline
6
  - make -C browser sha1
7
  - cp -p doc/analysis-example.vidjil browser/
8 9

stages:
10
  - test_tools
11
  - test_quality
12
  - test_unit
13
  - deploy_review
14
  - test_germlines
15
  - test_functional
16
  - test_shouldvdj
17
  - test_functional_external
18
  - coverage
19
  - prepare_release
20
  - tgz_release
Mathieu Giraud's avatar
Mathieu Giraud committed
21 22
  - valgrind_unit
  - valgrind_functional
23
  - multiple_tests
24
  - benchmark
25
  - publish_release
26
  - deploy_prod
27

28 29
# Anchors

30 31
.coverage_dependency:
  artifacts: &lcov_artifact
32 33
    paths:
      - algo/lcov_test_*.filtered
34
    expire_in: 1 week
35

36 37 38 39 40 41
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

42 43 44 45 46 47 48

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
49
    - make -C germline tests
50
  only:
51
    - /^feature-.*g.*\/.*$/
52
    - schedules
53

54 55 56 57
# Doc

include:
  - local: '/doc/.gitlab-ci.yml'
58
  - local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
59
  - local: 'algo/.gitlab-ci.yml' # Vidjil-algo pipelines
60

Mathieu Giraud's avatar
Mathieu Giraud committed
61
# Algorithm
62 63 64

test_algo_unit:
  stage: test_unit
65
  script: make -C algo unit_coverage
66 67
  artifacts:
    <<: *lcov_artifact
68
  only:
69
    - /^feature-.*a.*\/.*$/
70
    - schedules
71

72 73
test_algo_should:
  stage: test_functional
74
  script: make -C algo should_coverage
75
  artifacts:
76
    <<: *lcov_artifact
77 78
    reports:
      junit: algo/tests/should.xml
79
  only:
80
    - /^feature-.*a.*\/.*$/
81
    - schedules
82 83 84

test_algo_shouldvdj:
  stage: test_shouldvdj
85
  script: make -C algo shouldvdj_coverage
86 87
  artifacts:
    <<: *lcov_artifact
88 89
  only:
    - /^feature-.*a.*\/.*$/
90 91
    - schedules
  
92 93
algo_coverage:
  stage: coverage
94
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
95 96 97
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
98 99
  artifacts:
    paths:
100
      - coverage/
101
    expire_in: 1 week
102
  only:
103
    - /^feature-.*a.*\/.*$/
104
    - schedules
105

Mathieu Giraud's avatar
Mathieu Giraud committed
106 107
algo_valgrind_unit:
  stage: valgrind_unit
108
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
109
  only:
110
    - /^feature-.*a.*\/.*$/
111
    - schedules
112 113
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
114

115
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
116
  stage: valgrind_functional
117
  script: make -C algo valgrind_should
118 119 120 121 122
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
123 124
  when: manual
  only:
125
    - /^feature-.*a.*\/.*$/
126
    - schedules
127 128 129 130 131

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
132

133 134 135 136 137
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
138
    - /^feature-.*a.*\/.*$/
139

140 141
tgz_release:
  stage: tgz_release
142 143
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  only:
144
    - /^feature-.*a.*\/release$/
145
  artifacts:
146
    paths:
147 148
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
149
    expire_in: 10 years
150
  allow_failure: false
151 152
  tags:
    - ubuntu-16.04-amd64
153

154
copy_release:
155
  stage: publish_release
156 157 158 159 160 161 162 163
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
164
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
165
  when: manual
166 167 168
  only:
    - /^feature-.*a.*\/release$/

169 170
deploy_release_prod:
  stage: deploy_prod
171
  when: manual
172 173 174 175 176
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
177
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
178
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
179 180
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
181

Mathieu Giraud's avatar
Mathieu Giraud committed
182
# Client
183

184
test_browser_unit:
185
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
186
  script: make unit_browser
187 188 189
  artifacts:
    paths:
    - browser/
190
  only:
191 192
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
193
    - prod-client
194
    - dev
195
    - schedules
196 197
  tags:
    - web
198

199
.browser-functional:
200
  stage: test_functional
201
  retry: 2
202 203 204
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
205
    - rvm use 2.6.1
206
    - HEADLESS=1 make -C browser/test functional
207 208 209
  artifacts:
    paths:
    - browser/
210 211
    reports:
      junit: browser/test/test/TEST-*.xml
212
  only:
213 214
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
215
    - prod-client
216
    - schedules
217 218
  tags:
    - web
219

220 221 222 223 224
.browser-functional-chrome:
  extends: .browser-functional
  variables:
    WATIR_CHROME: "1"

225
.browser-functional-external:
226
  stage: test_functional_external
227
  retry: 2
228 229 230
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
231
    - rvm use 2.6.1
232
    - HEADLESS=1 make -C browser/test external
233 234 235
  artifacts:
    paths:
    - browser/
236 237
    reports:
      junit: browser/test/test/TEST-*.xml
238
  only:
239 240
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
241
    - prod-client
242
    - schedules
243 244
  tags:
    - web
245

246 247 248 249 250
.browser-functional-external-chrome:
  extends: .browser-functional-external
  variables:
    WATIR_CHROME: "1"
    
251
ff32-browser-functional:
252 253
  extends: .browser-functional
  variables:
254
    FIREFOX_VERSION: 32
255

256
ff45-browser-functional:
257 258
  extends: .browser-functional
  variables:
259 260
    FIREFOX_VERSION: 45

261
chrome-browser-functional:
262 263
  extends: .browser-functional-chrome
        
264
old-chrome-browser-functional:
265
  extends: .browser-functional-chrome
266 267 268
  tags:
    - legacy

269
ff32-browser-functional-external:
270 271
  extends: .browser-functional-external
  variables:
272
    FIREFOX_VERSION: 32
273

274
ff45-browser-functional-external:
275 276
  extends: .browser-functional-external
  variables:
277
    FIREFOX_VERSION: 45
278

279
chrome-browser-functional-external:
280 281
  extends: .browser-functional-external-chrome
    
282
old-chrome-browser-functional-external-chrome:
283 284 285
  extends: .browser-functional-external-chrome
  tags:
    - legacy
286

287 288
code_quality:
  stage: test_quality
289
  script: make -C browser quality
290
  only:
291 292
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
293
    - prod-client
294
    - dev
295
    - schedules
296 297
  tags:
    - web
298

Ryan Herbert's avatar
Ryan Herbert committed
299 300 301 302 303
# Server

test_server_unit:
    stage: test_unit
    script:
304 305
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
306
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
307
      - make -C server install_web2py_standalone
308
      - make -C server launch_fuse_server
309
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
310
      - make -C server kill_fuse_server
311
      - deactivate
312 313 314
    artifacts:
      reports:
        junit: server/web2py/test-reports/TEST*xml
Ryan Herbert's avatar
Ryan Herbert committed
315
    only:
316 317
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
Ryan Herbert's avatar
Ryan Herbert committed
318 319
      - prod-server
      - dev
320
      - schedules
Ryan Herbert's avatar
Ryan Herbert committed
321 322 323
    tags:
      - web

324
.test_server_functional:
325 326
  stage: test_functional
  script:
327 328
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
329
    - sed -i '/\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
330 331 332
    - sed -i '/\/var\/lib\/mysql/d' ./docker/docker-compose.yml
    - sed -i 's/\/opt\/vidjil\/databases/databases/' ./docker/docker-compose.yml
    - echo -e "volumes:\n    databases:" >> ./docker/docker-compose.yml
333
    - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml
334
    - cd docker/vidjil-server/conf/ && mv defs.py defs_https.py && mv defs_http.py defs.py && cd ../../..
335
    - make germline && cp browser/js/germline.js docker/vidjil-client/conf
336
    - mkdir -p docker/postfix/ssl
337
    - cd docker && docker-compose up -d && cd ..
338
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_tests.sh
339
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
340
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
341
    - source /etc/profile.d/rvm.sh
342
    - rvm use 2.6.1
343 344
    - HEADLESS=1 make functional_server
  after_script:
345
    - cd docker && docker-compose down -v
346
    - docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test
347 348 349
  artifacts:
    reports:
      junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml
350 351 352 353 354
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
355
    - schedules
356
  tags:
357
    - x86_64
358
    - docker
359

360
ff32-server-functional:
361 362 363 364
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 32

365
ff45-server-functional:
366 367 368 369 370
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 45


371 372 373 374

# Benchmark

benchmark_algo:
375
  image:  gcc:6.3
376
  extends: .install-algo-dependencies
377 378
  stage: benchmark
  script:
379
    - cd algo/tests ; python3 benchmark-releases.py -r 3 -bIc
380 381 382
  artifacts:
    paths:
      - algo/tests/benchmark.log
383 384 385 386
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
  tags:
387
    - cidocker
388 389


Ryan Herbert's avatar
Ryan Herbert committed
390 391
# Deployment

392 393
deploy_review:
  stage: deploy_review
394
  script:
395
    - make -C browser icons
396
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
397
    - rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG
398
  environment:
399
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
400
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
401
    on_stop: stop_deploy_review
402
  only:
403 404
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
405 406
  tags:
    - web
407

408 409 410
deploy_review_server:
  stage: deploy_review
  script:
411 412
    - make -C browser icons
    - cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
413
    - sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
414
    - sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
415
    - sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py
416 417
    - mv docker docker_${CI_BUILD_REF_SLUG}
    - cd docker_${CI_BUILD_REF_SLUG}
418 419
    - TMP_DOCKER=$(tempfile)
    - cp docker-compose.yml $TMP_DOCKER
420 421
    - awk -f ci/for-review.awk $TMP_DOCKER > docker-compose.yml
    - rm -f $TMP_DOCKER
422
    - sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml
423
    - cd ..
424
    - rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac)
425
    - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers"
426 427
  environment:
    name: review_server/$CI_BUILD_REF_NAME
428
    url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/
429
    on_stop: stop_deploy_review_server
430
    auto_stop_in: 2 months
431 432 433 434
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/

435
deploy_docker:
436
  stage: deploy_prod
437
  script:
438
    - export CUR_DATE=`date +%Y-%m-%d`
439
    - export TYPE=`echo $CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'`
440 441
    - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
    - export TAG="vidjil/$TYPE:$CUR_DATE-$SHA"
442
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
443
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE
444 445 446 447
    - docker push $TAG
    - docker tag $TAG "vidjil/$TYPE:latest"
    - docker push "vidjil/$TYPE:latest"
    - docker rmi $TAG "vidjil/$TYPE:latest"
448 449 450
  when: manual
  only:
    - prod-server
451
    - prod-client
452 453
  tags:
    - web
454
    - docker
455

456
deploy_prod:
457
  stage: deploy_prod
458
  script:
459 460 461 462 463 464 465
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
466 467
  environment:
    name: production
468
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
469 470
  only:
    - prod-client
471 472
  tags:
    - web
473

474 475 476 477 478
deploy_germlines:
  stage: deploy_prod
  script:
    - make -C germline get-all-data
    - make -C germline js
479
    - date=$(date +%Y-%m-%d)
480 481 482 483 484 485 486 487
    - tar cvzf germline-$date.tar.gz germline/*/*.fa germline/IMGT_RELEASE browser/js/germline.js
    - scp germline-$date.tar.gz $VIDJIL_WWW:$REMOTE_GERMLINE_DIR
  only:
    - manual
    - /^feature-.*g.*\/.*$/
    


488 489
stop_deploy_review:
  stage: deploy_review
490 491 492
  variables:
    GIT_STRATEGY: none
  script:
493
    - ssh $REVIEW_SERVER "rm -rf $REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG"
494 495
  when: manual
  environment:
496
    name: review/$CI_BUILD_REF_NAME
497
    action: stop
498
  only:
499 500
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
501
  tags:
502
    - web
503 504 505 506

stop_deploy_review_server:
  stage: deploy_review
  script:
507
    - ssh $REVIEW_SERVER "set -o xtrace; docker-compose -f $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG/docker_$CI_BUILD_REF_SLUG/docker-compose.yml down -v ; rm -rf $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG; $REVIEW_SERVER_PATH_SERVER/uninstall_certs.sh $CI_BUILD_REF_SLUG"
508 509 510 511 512 513 514
  when: manual
  environment:
    name: review_server/$CI_BUILD_REF_NAME
    action: stop
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/