.gitlab-ci.yml 8.03 KB
Newer Older
1 2 3


before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
4
  - make demo
5 6
  - make data
  - make germline
7
  - make -C browser sha1
8
  - cp -p doc/analysis-example.vidjil browser/
9 10

stages:
11
  - test_tools
12
  - test_quality
13
  - test_unit
14
  - deploy_review
15
  - test_germlines
16
  - test_functional
17
  - test_functional_external
18
  - test_shouldvdj
19
  - coverage
20
  - prepare_release
21
  - release
Mathieu Giraud's avatar
Mathieu Giraud committed
22 23
  - valgrind_unit
  - valgrind_functional
24
  - multiple_tests
25
  - prepare_deploy
26
  - deploy_docker
27
  - deploy_prod
28

29 30 31 32 33 34 35
# Anchors

.coverage_dependency: &lcov_artifact
  artifacts:
    paths:
      - algo/lcov_test_*.filtered
    expire_in: 6hrs
36

37 38 39 40 41 42
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

43 44 45 46 47 48 49

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
50
    - make -C germline tests
51
  only:
52
    - /^feature-.*g.*\/.*$/
53 54


Mathieu Giraud's avatar
Mathieu Giraud committed
55
# Algorithm
56 57 58

test_algo_unit:
  stage: test_unit
59 60
  script: make -C algo unit_coverage
  <<: *lcov_artifact
61
  only:
62
    - /^feature-.*a.*\/.*$/
63

64 65
test_algo_should:
  stage: test_functional
66
  script: make -C algo should_coverage
67
  only:
68
    - /^feature-.*a.*\/.*$/
69
  <<: *lcov_artifact
70 71 72

test_algo_shouldvdj:
  stage: test_shouldvdj
73 74
  script: make -C algo shouldvdj_coverage
  <<: *lcov_artifact
75 76 77 78 79
  only:
    - /^feature-.*a.*\/.*$/

algo_coverage:
  stage: coverage
80
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
81 82 83
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
84 85
  artifacts:
    paths:
86
      - coverage/
87
  only:
88
    - /^feature-.*a.*\/.*$/
89

Mathieu Giraud's avatar
Mathieu Giraud committed
90 91
algo_valgrind_unit:
  stage: valgrind_unit
92
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
93
  only:
94
    - /^feature-.*a.*\/.*$/
95 96
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
97

98
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
99
  stage: valgrind_functional
100
  script: make -C algo valgrind_should
101 102 103 104 105
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
106 107
  when: manual
  only:
108
    - /^feature-.*a.*\/.*$/
109 110 111 112 113

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
114

115 116 117 118 119
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
120
    - /^feature-.*a.*\/.*$/
121

122 123 124 125 126
release:
  stage: release
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  when: manual
  only:
127
    - /^feature-.*a.*\/release$/
128
  artifacts:
129
    paths:
130 131
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
132
  allow_failure: false
133 134
  tags:
    - ubuntu-16.04-amd64
135

136 137 138 139 140 141 142 143 144 145
copy_release:
  stage: prepare_deploy
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
146
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
147 148 149
  only:
    - /^feature-.*a.*\/release$/

150 151 152 153 154 155 156
deploy_release_prod:
  stage: deploy_prod
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
157
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
158
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
159 160
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
161

162 163
.testing-compilers: &test_various_compilers
  stage: multiple_tests
164 165 166 167 168 169 170 171 172 173 174 175
  tags:
    - several-compilers
  script:
    - |
      for CXX in $COMPILERS; do
        echo "**** Using $CXX for compilation ****"
        make -C algo clean
        make -C algo CXX=$CXX unit
        make -C algo CXX=$CXX should
        make -C algo CXX=$CXX valgrind_unit
      done

176 177 178 179 180 181 182 183 184 185 186
test_various_compilers_manual:
  <<: *test_various_compilers
  when: manual
  only:
    - /^feature-.*a.*\/.*$/

test_various_compilers_release:
  <<: *test_various_compilers
  only:
    - /^feature-.*a.*\/release$/

187

Mathieu Giraud's avatar
Mathieu Giraud committed
188
# Client
189

190
test_browser_unit:
191
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
192
  script: make unit_browser
193 194 195
  artifacts:
    paths:
    - browser/
196
  only:
197 198
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
199
    - prod-client
200
    - dev
201 202
  tags:
    - web
203

204 205
test_browser-functional:
  stage: test_functional
206 207 208 209
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
210 211 212 213 214
    - HEADLESS=1 make -C browser/test functional
  artifacts:
    paths:
    - browser/
  only:
215 216
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
217
    - prod-client
218 219
  tags:
    - web
220 221 222

test_browser-functional-external:
  stage: test_functional_external
223
  retry: 2
224 225 226 227 228
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
    - HEADLESS=1 make -C browser/test external
229 230 231
  artifacts:
    paths:
    - browser/
232
  only:
233 234
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
235
    - prod-client
236 237
  tags:
    - web
238

239 240
code_quality:
  stage: test_quality
241
  script: make -C browser quality
242
  only:
243 244
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
245
    - prod-client
246
    - dev
247 248
  tags:
    - web
249

Ryan Herbert's avatar
Ryan Herbert committed
250 251 252 253 254
# Server

test_server_unit:
    stage: test_unit
    script:
255 256
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
257
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
258
      - make -C server install_web2py_standalone
259
      - make -C server launch_fuse_server
260
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
261
      - make -C server kill_fuse_server
262
      - deactivate
Ryan Herbert's avatar
Ryan Herbert committed
263
    only:
264 265
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
Ryan Herbert's avatar
Ryan Herbert committed
266 267 268 269 270
      - prod-server
      - dev
    tags:
      - web

271 272 273
test_server_functional:
  stage: test_functional
  script:
274 275
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
276
    - cd docker && docker-compose up -d && cd ..
Ryan Herbert's avatar
Ryan Herbert committed
277
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
278
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
279
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
280 281
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
282
    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; false)
283
    - cd docker && docker-compose stop
284 285 286 287 288 289
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
  tags:
290
    - x86_64
291
    - docker
292

Ryan Herbert's avatar
Ryan Herbert committed
293 294
# Deployment

295 296
deploy_review:
  stage: deploy_review
297
  script:
298
    - make -C browser icons
299
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
300
    - rsync -av --delete browser $DEPLOY_SERVER:$DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG
301
  environment:
302
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
303
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
304
    on_stop: stop_deploy_review
305
  only:
306 307
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
308 309
  tags:
    - web
310

311 312 313 314 315 316 317 318 319 320
deploy_docker:
  stage: deploy_docker
  script:
    - docker tag $CI_BUILD_REF_SLUG vidjil/vidjil:latest
    - docker push vidjil/vidjil:latest
  when: manual
  only:
    - prod-server
  tags:
    - web
321
    - docker
322

323
deploy_prod:
324
  stage: deploy_prod
325
  script:
326 327 328 329 330 331 332
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
333 334
  environment:
    name: production
335
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
336 337
  only:
    - prod-client
338 339
  tags:
    - web
340

341 342
stop_deploy_review:
  stage: deploy_review
343 344 345 346 347 348
  variables:
    GIT_STRATEGY: none
  script:
    - ssh $DEPLOY_SERVER "rm -rf $DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG"
  when: manual
  environment:
349
    name: review/$CI_BUILD_REF_NAME
350
    action: stop
351
  only:
352 353
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
354
  tags:
355
    - web