.gitlab-ci.yml 8.99 KB
Newer Older
1 2 3


before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
4
  - make demo
5 6
  - make data
  - make germline
7
  - make -C browser sha1
8
  - cp -p doc/analysis-example.vidjil browser/
9 10

stages:
11
  - test_tools
12
  - test_quality
13
  - test_unit
14
  - deploy_review
15
  - test_germlines
16
  - test_shouldvdj
17 18
  - test_functional
  - test_functional_external
19
  - coverage
20
  - prepare_release
21
  - release
Mathieu Giraud's avatar
Mathieu Giraud committed
22 23
  - valgrind_unit
  - valgrind_functional
24
  - multiple_tests
25
  - prepare_deploy
26
  - deploy_prod
27

28 29
# Anchors

30 31
.coverage_dependency:
  artifacts: &lcov_artifact
32 33 34
    paths:
      - algo/lcov_test_*.filtered
    expire_in: 6hrs
35

36 37 38 39 40 41
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

42 43 44 45 46 47 48

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
49
    - make -C germline tests
50
  only:
51
    - /^feature-.*g.*\/.*$/
52
    - schedules
53 54


Mathieu Giraud's avatar
Mathieu Giraud committed
55
# Algorithm
56 57 58

test_algo_unit:
  stage: test_unit
59
  script: make -C algo unit_coverage
60 61
  artifacts:
    <<: *lcov_artifact
62
  only:
63
    - /^feature-.*a.*\/.*$/
64
    - schedules
65

66 67
test_algo_should:
  stage: test_functional
68
  script: make -C algo should_coverage
69
  artifacts:
70
    <<: *lcov_artifact
71 72
    reports:
      junit: algo/tests/should.xml
73
  only:
74
    - /^feature-.*a.*\/.*$/
75
    - schedules
76 77 78

test_algo_shouldvdj:
  stage: test_shouldvdj
79
  script: make -C algo shouldvdj_coverage
80 81
  artifacts:
    <<: *lcov_artifact
82 83
  only:
    - /^feature-.*a.*\/.*$/
84 85
    - schedules
  
86 87
algo_coverage:
  stage: coverage
88
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
89 90 91
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
92 93
  artifacts:
    paths:
94
      - coverage/
95
  only:
96
    - /^feature-.*a.*\/.*$/
97
    - schedules
98

Mathieu Giraud's avatar
Mathieu Giraud committed
99 100
algo_valgrind_unit:
  stage: valgrind_unit
101
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
102
  only:
103
    - /^feature-.*a.*\/.*$/
104
    - schedules
105 106
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
107

108
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
109
  stage: valgrind_functional
110
  script: make -C algo valgrind_should
111 112 113 114 115
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
116 117
  when: manual
  only:
118
    - /^feature-.*a.*\/.*$/
119
    - schedules
120 121 122 123 124

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
125

126 127 128 129 130
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
131
    - /^feature-.*a.*\/.*$/
132

133 134 135 136 137
release:
  stage: release
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  when: manual
  only:
138
    - /^feature-.*a.*\/release$/
139
  artifacts:
140
    paths:
141 142
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
143
  allow_failure: false
144 145
  tags:
    - ubuntu-16.04-amd64
146

147 148 149 150 151 152 153 154 155 156
copy_release:
  stage: prepare_deploy
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
157
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
158 159 160
  only:
    - /^feature-.*a.*\/release$/

161 162 163 164 165 166 167
deploy_release_prod:
  stage: deploy_prod
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
168
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
169
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
170 171
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
172

173 174
.testing-compilers: &test_various_compilers
  stage: multiple_tests
175 176 177 178 179 180 181 182 183 184 185 186
  tags:
    - several-compilers
  script:
    - |
      for CXX in $COMPILERS; do
        echo "**** Using $CXX for compilation ****"
        make -C algo clean
        make -C algo CXX=$CXX unit
        make -C algo CXX=$CXX should
        make -C algo CXX=$CXX valgrind_unit
      done

187 188 189 190 191
test_various_compilers_manual:
  <<: *test_various_compilers
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
192
    - schedules
193 194 195 196 197 198

test_various_compilers_release:
  <<: *test_various_compilers
  only:
    - /^feature-.*a.*\/release$/

199

Mathieu Giraud's avatar
Mathieu Giraud committed
200
# Client
201

202
test_browser_unit:
203
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
204
  script: make unit_browser
205 206 207
  artifacts:
    paths:
    - browser/
208
  only:
209 210
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
211
    - prod-client
212
    - dev
213
    - schedules
214 215
  tags:
    - web
216

217 218
test_browser-functional:
  stage: test_functional
219
  retry: 2
220 221 222 223
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
224 225 226 227 228
    - HEADLESS=1 make -C browser/test functional
  artifacts:
    paths:
    - browser/
  only:
229 230
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
231
    - prod-client
232
    - schedules
233 234
  tags:
    - web
235 236 237

test_browser-functional-external:
  stage: test_functional_external
238
  retry: 2
239 240 241 242 243
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
    - HEADLESS=1 make -C browser/test external
244 245 246
  artifacts:
    paths:
    - browser/
247
  only:
248 249
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
250
    - prod-client
251
    - schedules
252 253
  tags:
    - web
254

255 256
code_quality:
  stage: test_quality
257
  script: make -C browser quality
258
  only:
259 260
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
261
    - prod-client
262
    - dev
263
    - schedules
264 265
  tags:
    - web
266

Ryan Herbert's avatar
Ryan Herbert committed
267 268 269 270 271
# Server

test_server_unit:
    stage: test_unit
    script:
272 273
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
274
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
275
      - make -C server install_web2py_standalone
276
      - make -C server launch_fuse_server
277
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
278
      - make -C server kill_fuse_server
279
      - deactivate
Ryan Herbert's avatar
Ryan Herbert committed
280
    only:
281 282
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
Ryan Herbert's avatar
Ryan Herbert committed
283 284
      - prod-server
      - dev
285
      - schedules
Ryan Herbert's avatar
Ryan Herbert committed
286 287 288
    tags:
      - web

289 290 291
test_server_functional:
  stage: test_functional
  script:
292 293
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
Ryan Herbert's avatar
Ryan Herbert committed
294
    - sed -i '/\/etc\/nginx\/ssl\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
295
    - sed -i 's/\:latest/\:test/g' ./docker/docker-compose.yml
296
    - cd docker && docker-compose up -d && cd ..
Ryan Herbert's avatar
Ryan Herbert committed
297
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
298
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
299
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
300 301
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
302
    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; false)
303
    - cd docker && docker-compose stop
304
    - docker rmi "vidjil/server:test" "vidjil/client:test"
305 306 307 308 309
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
310
    - schedules
311
  tags:
312
    - x86_64
313
    - docker
314

Ryan Herbert's avatar
Ryan Herbert committed
315 316
# Deployment

317 318
deploy_review:
  stage: deploy_review
319
  script:
320
    - make -C browser icons
321
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
322
    - rsync -av --delete browser $DEPLOY_SERVER:$DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG
323
  environment:
324
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
325
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
326
    on_stop: stop_deploy_review
327
  only:
328 329
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
330 331
  tags:
    - web
332

333
deploy_docker:
334
  stage: deploy_prod
335
  script:
336 337 338
    - export CUR_DATE=date +%Y-%m-%d
    - export TYPE=$CI_COMMIT_REF_NAME | awk '{ gsub(/prod-/, "") }{ print $1 }'
    - export TAG="vidjil/$TYPE:$CI_COMMIT_SHA-$CUR_DATE"
339
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=PRODUCTION -t $TAG docker/vidjil-$TYPE
340 341 342 343 344
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - docker push $TAG
    - docker tag $TAG "vidjil/$TYPE:latest"
    - docker push "vidjil/$TYPE:latest"
    - docker rmi $TAG "vidjil/$TYPE:latest"
345 346 347
  when: manual
  only:
    - prod-server
348
    - prod-client
349 350
  tags:
    - web
351
    - docker
352

353
deploy_prod:
354
  stage: deploy_prod
355
  script:
356 357 358 359 360 361 362
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
363 364
  environment:
    name: production
365
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
366 367
  only:
    - prod-client
368 369
  tags:
    - web
370

371 372
stop_deploy_review:
  stage: deploy_review
373 374 375 376 377 378
  variables:
    GIT_STRATEGY: none
  script:
    - ssh $DEPLOY_SERVER "rm -rf $DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG"
  when: manual
  environment:
379
    name: review/$CI_BUILD_REF_NAME
380
    action: stop
381
  only:
382 383
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
384
  tags:
385
    - web