.gitlab-ci.yml 10 KB
Newer Older
1 2

before_script:
3
  - make demo
4 5
  - make data
  - make germline
6
  - make -C browser sha1
7
  - cp -p doc/analysis-example.vidjil browser/
8 9

stages:
10
  - test_tools
11
  - test_quality
12
  - test_unit
13
  - deploy_review
14
  - test_germlines
15
  - test_functional
16
  - test_shouldvdj
17
  - test_functional_external
18
  - coverage
19
  - prepare_release
20
  - tgz_release
21 22
  - valgrind_unit
  - valgrind_functional
23
  - multiple_tests
24
  - publish_release
25
  - deploy_prod
26

27 28
# Anchors

29 30
.coverage_dependency:
  artifacts: &lcov_artifact
31 32
    paths:
      - algo/lcov_test_*.filtered
33
    expire_in: 1 week
34

35 36 37 38 39 40
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

41 42 43 44 45 46 47

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
48
    - make -C germline tests
49
  only:
50
    - /^feature-.*g.*\/.*$/
51
    - schedules
52 53


54
# Algorithm
55 56 57

test_algo_unit:
  stage: test_unit
58
  script: make -C algo unit_coverage
59 60
  artifacts:
    <<: *lcov_artifact
61
  only:
62
    - /^feature-.*a.*\/.*$/
63
    - schedules
64

65 66
test_algo_should:
  stage: test_functional
67
  script: make -C algo should_coverage
68
  artifacts:
69
    <<: *lcov_artifact
70 71
    reports:
      junit: algo/tests/should.xml
72
  only:
73
    - /^feature-.*a.*\/.*$/
74
    - schedules
75 76 77

test_algo_shouldvdj:
  stage: test_shouldvdj
78
  script: make -C algo shouldvdj_coverage
79 80
  artifacts:
    <<: *lcov_artifact
81 82
  only:
    - /^feature-.*a.*\/.*$/
83 84
    - schedules
  
85 86
algo_coverage:
  stage: coverage
87
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
88 89 90
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
91 92
  artifacts:
    paths:
93
      - coverage/
94
    expire_in: 1 week
95
  only:
96
    - /^feature-.*a.*\/.*$/
97
    - schedules
98

99 100
algo_valgrind_unit:
  stage: valgrind_unit
101
  script: make -C algo valgrind_unit
102
  only:
103
    - /^feature-.*a.*\/.*$/
104
    - schedules
105 106
  tags:
    - valgrind
107

108
.snip_valgrind_functional: &valgrind_functional
109
  stage: valgrind_functional
110
  script: make -C algo valgrind_should
111 112 113 114 115
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
116 117
  when: manual
  only:
118
    - /^feature-.*a.*\/.*$/
119
    - schedules
120 121 122 123 124

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
125

126 127 128 129 130
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
131
    - /^feature-.*a.*\/.*$/
132

133 134
tgz_release:
  stage: tgz_release
135 136
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  only:
137
    - /^feature-.*a.*\/release$/
138
  artifacts:
139
    paths:
140 141
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
142
    expire_in: 10 years
143
  allow_failure: false
144 145
  tags:
    - ubuntu-16.04-amd64
146

147
copy_release:
148
  stage: publish_release
149 150 151 152 153 154 155 156
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
157
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
158
  when: manual
159 160 161
  only:
    - /^feature-.*a.*\/release$/

162 163
deploy_release_prod:
  stage: deploy_prod
164
  when: manual
165 166 167 168 169
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
170
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
171
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
172 173
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
174

175 176
.testing-compilers: &test_various_compilers
  stage: multiple_tests
177 178 179 180 181 182 183 184 185 186 187 188
  tags:
    - several-compilers
  script:
    - |
      for CXX in $COMPILERS; do
        echo "**** Using $CXX for compilation ****"
        make -C algo clean
        make -C algo CXX=$CXX unit
        make -C algo CXX=$CXX should
        make -C algo CXX=$CXX valgrind_unit
      done

189 190 191 192 193
test_various_compilers_manual:
  <<: *test_various_compilers
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
194
    - schedules
195 196 197 198 199 200

test_various_compilers_release:
  <<: *test_various_compilers
  only:
    - /^feature-.*a.*\/release$/

201

202
# Client
203

204
test_browser_unit:
205
  stage: test_unit
206
  script: make unit_browser
207 208 209
  artifacts:
    paths:
    - browser/
210
  only:
211 212
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
213
    - prod-client
214
    - dev
215
    - schedules
216 217
  tags:
    - web
218

219 220
test_browser-functional:
  stage: test_functional
221
  retry: 2
222 223 224 225
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
226 227 228 229 230
    - HEADLESS=1 make -C browser/test functional
  artifacts:
    paths:
    - browser/
  only:
231 232
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
233
    - prod-client
234
    - schedules
235 236
  tags:
    - web
237 238 239

test_browser-functional-external:
  stage: test_functional_external
240
  retry: 2
241 242 243 244 245
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
    - HEADLESS=1 make -C browser/test external
246 247 248
  artifacts:
    paths:
    - browser/
249
  only:
250 251
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
252
    - prod-client
253
    - schedules
254 255
  tags:
    - web
256

257 258
code_quality:
  stage: test_quality
259
  script: make -C browser quality
260
  only:
261 262
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
263
    - prod-client
264
    - dev
265
    - schedules
266 267
  tags:
    - web
268

269 270 271 272 273
# Server

test_server_unit:
    stage: test_unit
    script:
274 275
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
276
      - pip install -r requirements.txt
277
      - make -C server install_web2py_standalone
278
      - make -C server launch_fuse_server
279
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
280
      - make -C server kill_fuse_server
281
      - deactivate
282 283 284
    artifacts:
      reports:
        junit: server/web2py/test-reports/TEST*xml
285
    only:
286 287
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
288 289
      - prod-server
      - dev
290
      - schedules
291 292 293
    tags:
      - web

294 295 296
test_server_functional:
  stage: test_functional
  script:
297 298
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
299
    - sed -i '/\/etc\/nginx\/ssl\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
300
    - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml
301 302
    - cd docker/vidjil-server/conf/ && mv defs.py defs_https.py && mv defs_http.py defs.py && cd ../../..
    - cd docker/vidjil-client/conf/ && mv conf.js conf_https.js && mv conf_http.js conf.js && cd ../../..
303
    - make germline && cp browser/js/germline.js docker/vidjil-client/conf
304
    - cd docker && docker-compose up -d && cd ..
305
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
306
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
307
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
308 309
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
310
    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; docker stop $(docker ps -aq); docker rm $(docker ps -aq); docker rmi "vidjil/server:test" "vidjil/client:test"; false)
311
    - cd docker && docker-compose stop
312 313
    - docker stop $(docker ps -aq)
    - docker rm $(docker ps -aq)
314
    - docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test
315 316 317 318 319
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
320
    - schedules
321
  tags:
322
    - x86_64
323
    - docker
324

325 326
# Deployment

327 328
deploy_review:
  stage: deploy_review
329
  script:
330
    - make -C browser icons
331
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
332
    - rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG
333
  environment:
334
    name: review/$CI_BUILD_REF_NAME
335
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
336
    on_stop: stop_deploy_review
337
  only:
338 339
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
340 341
  tags:
    - web
342

343
deploy_docker:
344
  stage: deploy_prod
345
  script:
346
    - export CUR_DATE=`date +%Y-%m-%d`
347
    - export TYPE=`echo $CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'`
348 349
    - export SHA=`echo $CI_COMMIT_SHA | awk '{ print substr($0, 0, 8) }'`
    - export TAG="vidjil/$TYPE:$CUR_DATE-$SHA"
350
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
351
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE
352 353 354 355
    - docker push $TAG
    - docker tag $TAG "vidjil/$TYPE:latest"
    - docker push "vidjil/$TYPE:latest"
    - docker rmi $TAG "vidjil/$TYPE:latest"
356 357 358
  when: manual
  only:
    - prod-server
359
    - prod-client
360 361
  tags:
    - web
362
    - docker
363

364
deploy_prod:
365
  stage: deploy_prod
366
  script:
367 368 369 370 371 372 373
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
374 375
  environment:
    name: production
376
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
377 378
  only:
    - prod-client
379 380
  tags:
    - web
381

382 383 384 385 386
deploy_germlines:
  stage: deploy_prod
  script:
    - make -C germline get-all-data
    - make -C germline js
387
    - date=$(date +%Y-%m-%d)
388 389 390 391 392 393 394 395
    - tar cvzf germline-$date.tar.gz germline/*/*.fa germline/IMGT_RELEASE browser/js/germline.js
    - scp germline-$date.tar.gz $VIDJIL_WWW:$REMOTE_GERMLINE_DIR
  only:
    - manual
    - /^feature-.*g.*\/.*$/
    


396 397
stop_deploy_review:
  stage: deploy_review
398 399 400
  variables:
    GIT_STRATEGY: none
  script:
401
    - ssh $REVIEW_SERVER "rm -rf $REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG"
402 403
  when: manual
  environment:
404
    name: review/$CI_BUILD_REF_NAME
405
    action: stop
406
  only:
407 408
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
409
  tags:
410
    - web