.gitlab-ci.yml 11 KB
Newer Older
1 2

before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
3
  - make demo
4 5
  - make data
  - make germline
6
  - make -C browser sha1
7
  - cp -p doc/analysis-example.vidjil browser/
8 9

stages:
10
  - test_tools
11
  - test_quality
12
  - test_unit
13
  - deploy_review
14
  - test_germlines
15
  - test_functional
16
  - test_shouldvdj
17
  - test_functional_external
18
  - coverage
19
  - prepare_release
20
  - tgz_release
Mathieu Giraud's avatar
Mathieu Giraud committed
21 22
  - valgrind_unit
  - valgrind_functional
23
  - multiple_tests
24
  - benchmark
25
  - publish_release
26
  - deploy_prod
27

28 29
# Anchors

30 31
.coverage_dependency:
  artifacts: &lcov_artifact
32 33
    paths:
      - algo/lcov_test_*.filtered
34
    expire_in: 1 week
35

36 37 38 39 40 41
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

42 43 44 45 46 47 48

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
49
    - make -C germline tests
50
  only:
51
    - /^feature-.*g.*\/.*$/
52
    - schedules
53

54 55 56 57
# Doc

include:
  - local: '/doc/.gitlab-ci.yml'
58
  - local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
59

Mathieu Giraud's avatar
Mathieu Giraud committed
60
# Algorithm
61 62 63

test_algo_unit:
  stage: test_unit
64
  script: make -C algo unit_coverage
65 66
  artifacts:
    <<: *lcov_artifact
67
  only:
68
    - /^feature-.*a.*\/.*$/
69
    - schedules
70

71 72
test_algo_should:
  stage: test_functional
73
  script: make -C algo should_coverage
74
  artifacts:
75
    <<: *lcov_artifact
76 77
    reports:
      junit: algo/tests/should.xml
78
  only:
79
    - /^feature-.*a.*\/.*$/
80
    - schedules
81 82 83

test_algo_shouldvdj:
  stage: test_shouldvdj
84
  script: make -C algo shouldvdj_coverage
85 86
  artifacts:
    <<: *lcov_artifact
87 88
  only:
    - /^feature-.*a.*\/.*$/
89 90
    - schedules
  
91 92
algo_coverage:
  stage: coverage
93
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
94 95 96
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
97 98
  artifacts:
    paths:
99
      - coverage/
100
    expire_in: 1 week
101
  only:
102
    - /^feature-.*a.*\/.*$/
103
    - schedules
104

Mathieu Giraud's avatar
Mathieu Giraud committed
105 106
algo_valgrind_unit:
  stage: valgrind_unit
107
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
108
  only:
109
    - /^feature-.*a.*\/.*$/
110
    - schedules
111 112
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
113

114
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
115
  stage: valgrind_functional
116
  script: make -C algo valgrind_should
117 118 119 120 121
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
122 123
  when: manual
  only:
124
    - /^feature-.*a.*\/.*$/
125
    - schedules
126 127 128 129 130

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
131

132 133 134 135 136
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
137
    - /^feature-.*a.*\/.*$/
138

139 140
tgz_release:
  stage: tgz_release
141 142
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  only:
143
    - /^feature-.*a.*\/release$/
144
  artifacts:
145
    paths:
146 147
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
148
    expire_in: 10 years
149
  allow_failure: false
150 151
  tags:
    - ubuntu-16.04-amd64
152

153
copy_release:
154
  stage: publish_release
155 156 157 158 159 160 161 162
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
163
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
164
  when: manual
165 166 167
  only:
    - /^feature-.*a.*\/release$/

168 169
deploy_release_prod:
  stage: deploy_prod
170
  when: manual
171 172 173 174 175
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
176
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
177
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
178 179
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
180

Mathieu Giraud's avatar
Mathieu Giraud committed
181
# Client
182

183
test_browser_unit:
184
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
185
  script: make unit_browser
186 187 188
  artifacts:
    paths:
    - browser/
189
  only:
190 191
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
192
    - prod-client
193
    - dev
194
    - schedules
195 196
  tags:
    - web
197

198
.browser-functional:
199
  stage: test_functional
200
  retry: 2
201 202 203
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
204
    - rvm use 2.6.1
205
    - HEADLESS=1 make -C browser/test functional
206 207 208
  artifacts:
    paths:
    - browser/
209 210
    reports:
      junit: browser/test/test/TEST-*.xml
211
  only:
212 213
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
214
    - prod-client
215
    - schedules
216 217
  tags:
    - web
218

219 220 221 222 223
.browser-functional-chrome:
  extends: .browser-functional
  variables:
    WATIR_CHROME: "1"

224
.browser-functional-external:
225
  stage: test_functional_external
226
  retry: 2
227 228 229
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
230
    - rvm use 2.6.1
231
    - HEADLESS=1 make -C browser/test external
232 233 234
  artifacts:
    paths:
    - browser/
235 236
    reports:
      junit: browser/test/test/TEST-*.xml
237
  only:
238 239
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
240
    - prod-client
241
    - schedules
242 243
  tags:
    - web
244

245 246 247 248 249
.browser-functional-external-chrome:
  extends: .browser-functional-external
  variables:
    WATIR_CHROME: "1"
    
250
ff32-browser-functional:
251 252
  extends: .browser-functional
  variables:
253
    FIREFOX_VERSION: 32
254

255
ff45-browser-functional:
256 257
  extends: .browser-functional
  variables:
258 259
    FIREFOX_VERSION: 45

260
chrome-browser-functional:
261 262
  extends: .browser-functional-chrome
        
263
old-chrome-browser-functional:
264
  extends: .browser-functional-chrome
265 266 267
  tags:
    - legacy

268
ff32-browser-functional-external:
269 270
  extends: .browser-functional-external
  variables:
271
    FIREFOX_VERSION: 32
272

273
ff45-browser-functional-external:
274 275
  extends: .browser-functional-external
  variables:
276
    FIREFOX_VERSION: 45
277

278
chrome-browser-functional-external:
279 280
  extends: .browser-functional-external-chrome
    
281
old-chrome-browser-functional-external-chrome:
282 283 284
  extends: .browser-functional-external-chrome
  tags:
    - legacy
285

286 287
code_quality:
  stage: test_quality
288
  script: make -C browser quality
289
  only:
290 291
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
292
    - prod-client
293
    - dev
294
    - schedules
295 296
  tags:
    - web
297

Ryan Herbert's avatar
Ryan Herbert committed
298 299 300 301 302
# Server

test_server_unit:
    stage: test_unit
    script:
303 304
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
305
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
306
      - make -C server install_web2py_standalone
307
      - make -C server launch_fuse_server
308
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
309
      - make -C server kill_fuse_server
310
      - deactivate
311 312 313
    artifacts:
      reports:
        junit: server/web2py/test-reports/TEST*xml
Ryan Herbert's avatar
Ryan Herbert committed
314
    only:
315 316
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
Ryan Herbert's avatar
Ryan Herbert committed
317 318
      - prod-server
      - dev
319
      - schedules
Ryan Herbert's avatar
Ryan Herbert committed
320 321 322
    tags:
      - web

323
.test_server_functional:
324 325
  stage: test_functional
  script:
326 327
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
Ryan Herbert's avatar
Ryan Herbert committed
328
    - sed -i '/\/etc\/nginx\/ssl\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
329
    - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml
330
    - cd docker/vidjil-server/conf/ && mv defs.py defs_https.py && mv defs_http.py defs.py && cd ../../..
331
    - make germline && cp browser/js/germline.js docker/vidjil-client/conf
332
    - cd docker && docker-compose up -d && cd ..
333
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_tests.sh
334
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
335
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
336
    - source /etc/profile.d/rvm.sh
337
    - rvm use 2.6.1
338
    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; docker stop $(docker ps -aq); docker rm $(docker ps -aq); docker rmi "vidjil/server:test" "vidjil/client:test"; false)
339
    - cd docker && docker-compose stop
340 341
    - docker stop $(docker ps -aq)
    - docker rm $(docker ps -aq)
342
    - docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test
343 344 345
  artifacts:
    reports:
      junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml
346 347 348 349 350
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
351
    - schedules
352
  tags:
353
    - x86_64
354
    - docker
355

356
ff32-server-functional:
357 358 359 360
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 32

361
ff45-server-functional:
362 363 364 365 366
  extends: .test_server_functional
  variables:
    FIREFOX_VERSION: 45


367 368 369 370 371 372

# Benchmark

benchmark_algo:
  stage: benchmark
  script:
373
    - cd algo/tests ; python3 benchmark-releases.py -bic
374 375 376 377 378 379 380
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
  tags:
    - several-compilers


Ryan Herbert's avatar
Ryan Herbert committed
381 382
# Deployment

383 384
deploy_review:
  stage: deploy_review
385
  script:
386
    - make -C browser icons
387
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
388
    - rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG
389
  environment:
390
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
391
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
392
    on_stop: stop_deploy_review
393
  only:
394 395
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
396 397
  tags:
    - web
398

399
deploy_docker:
400
  stage: deploy_prod
401
  script:
402
    - export CUR_DATE=`date +%Y-%m-%d`
403
    - export TYPE=`echo $CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'`
404 405
    - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
    - export TAG="vidjil/$TYPE:$CUR_DATE-$SHA"
406
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
407
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE
408 409 410 411
    - docker push $TAG
    - docker tag $TAG "vidjil/$TYPE:latest"
    - docker push "vidjil/$TYPE:latest"
    - docker rmi $TAG "vidjil/$TYPE:latest"
412 413 414
  when: manual
  only:
    - prod-server
415
    - prod-client
416 417
  tags:
    - web
418
    - docker
419

420
deploy_prod:
421
  stage: deploy_prod
422
  script:
423 424 425 426 427 428 429
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
430 431
  environment:
    name: production
432
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
433 434
  only:
    - prod-client
435 436
  tags:
    - web
437

438 439 440 441 442
deploy_germlines:
  stage: deploy_prod
  script:
    - make -C germline get-all-data
    - make -C germline js
443
    - date=$(date +%Y-%m-%d)
444 445 446 447 448 449 450 451
    - tar cvzf germline-$date.tar.gz germline/*/*.fa germline/IMGT_RELEASE browser/js/germline.js
    - scp germline-$date.tar.gz $VIDJIL_WWW:$REMOTE_GERMLINE_DIR
  only:
    - manual
    - /^feature-.*g.*\/.*$/
    


452 453
stop_deploy_review:
  stage: deploy_review
454 455 456
  variables:
    GIT_STRATEGY: none
  script:
457
    - ssh $REVIEW_SERVER "rm -rf $REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG"
458 459
  when: manual
  environment:
460
    name: review/$CI_BUILD_REF_NAME
461
    action: stop
462
  only:
463 464
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
465
  tags:
466
    - web