.gitlab-ci.yml 8.35 KB
Newer Older
1
2
3


before_script:
Mathieu Giraud's avatar
Mathieu Giraud committed
4
  - make demo
5
6
  - make data
  - make germline
7
  - make -C browser sha1
8
  - cp -p doc/analysis-example.vidjil browser/
9
10

stages:
11
  - test_tools
12
  - test_quality
13
  - test_unit
14
  - deploy_review
15
  - test_germlines
16
  - test_shouldvdj
17
18
  - test_functional
  - test_functional_external
19
  - coverage
20
  - prepare_release
21
  - release
Mathieu Giraud's avatar
Mathieu Giraud committed
22
23
  - valgrind_unit
  - valgrind_functional
24
  - multiple_tests
25
  - prepare_deploy
26
  - deploy_docker
27
  - deploy_prod
28

29
30
# Anchors

31
32
.coverage_dependency:
  artifacts: &lcov_artifact
33
34
35
    paths:
      - algo/lcov_test_*.filtered
    expire_in: 6hrs
36

37
38
39
40
41
42
# Tools

test_tools:
  stage: test_tools
  script: make -C tools/tests

43
44
45
46
47
48
49

# Germlines

test_germlines:
  stage: test_germlines
  script:
    - make -C germline get-all-data
50
    - make -C germline tests
51
  only:
52
    - /^feature-.*g.*\/.*$/
53
    - schedules
54
55


Mathieu Giraud's avatar
Mathieu Giraud committed
56
# Algorithm
57
58
59

test_algo_unit:
  stage: test_unit
60
  script: make -C algo unit_coverage
61
62
  artifacts:
    <<: *lcov_artifact
63
  only:
64
    - /^feature-.*a.*\/.*$/
65
    - schedules
66

67
68
test_algo_should:
  stage: test_functional
69
  script: make -C algo should_coverage
70
  artifacts:
71
    <<: *lcov_artifact
72
73
    reports:
      junit: algo/tests/should.xml
74
  only:
75
    - /^feature-.*a.*\/.*$/
76
    - schedules
77
78
79

test_algo_shouldvdj:
  stage: test_shouldvdj
80
  script: make -C algo shouldvdj_coverage
81
82
  artifacts:
    <<: *lcov_artifact
83
84
  only:
    - /^feature-.*a.*\/.*$/
85
86
    - schedules
  
87
88
algo_coverage:
  stage: coverage
89
  coverage: /^\s*lines\.*:\s+([0-9.]+\%)/
90
91
92
  script:
    - make -C algo lcov_reports
    - mv algo/reports/ coverage
93
94
  artifacts:
    paths:
95
      - coverage/
96
  only:
97
    - /^feature-.*a.*\/.*$/
98
    - schedules
99

Mathieu Giraud's avatar
Mathieu Giraud committed
100
101
algo_valgrind_unit:
  stage: valgrind_unit
102
  script: make -C algo valgrind_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
103
  only:
104
    - /^feature-.*a.*\/.*$/
105
    - schedules
106
107
  tags:
    - valgrind
Mathieu Giraud's avatar
Mathieu Giraud committed
108

109
.snip_valgrind_functional: &valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
110
  stage: valgrind_functional
111
  script: make -C algo valgrind_should
112
113
114
115
116
  tags:
    - valgrind

algo_valgrind_functional:
  <<: *valgrind_functional
Mathieu Giraud's avatar
Mathieu Giraud committed
117
118
  when: manual
  only:
119
    - /^feature-.*a.*\/.*$/
120
    - schedules
121
122
123
124
125

release_valgrind_functional:
  <<: *valgrind_functional
  only:
    - /^feature-.*a.*\/release$/
Mathieu Giraud's avatar
Mathieu Giraud committed
126

127
128
129
130
131
prepare_release:
  stage: prepare_release
  script: make -C algo release RELEASE_TAG='rc'
  when: manual
  only:
132
    - /^feature-.*a.*\/.*$/
133

134
135
136
137
138
release:
  stage: release
  script: make -C algo release RELEASE_TAG=`cat algo/release`
  when: manual
  only:
139
    - /^feature-.*a.*\/release$/
140
  artifacts:
141
    paths:
142
143
    - algo/releases/vidjil-algo-`cat algo/release`.tar.gz
    - algo/releases/vidjil-algo-`cat algo/release`/vidjil-algo-`cat algo/release`_`uname -m`
144
  allow_failure: false
145
146
  tags:
    - ubuntu-16.04-amd64
147

148
149
150
151
152
153
154
155
156
157
copy_release:
  stage: prepare_deploy
  script:
    - |
      for repo in $ALGO_REPOSITORIES; do
        echo "Copying release to $repo"
        scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
      done
      release_name=$(cat algo/release)
      ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
Mikaël Salson's avatar
Mikaël Salson committed
158
      ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
159
160
161
  only:
    - /^feature-.*a.*\/release$/

162
163
164
165
166
167
168
deploy_release_prod:
  stage: deploy_prod
  only:
    - /^feature-.*a.*\/release$/
  script:
    - |
      release_name=$(cat algo/release)
169
      ssh $DEPLOY_SERVER "cd releases/vidjil;\
170
          tar xvzf vidjil-algo-${release_name}.tar.gz;\
171
172
          make -C vidjil-algo-${release_name};\
          ln -sfT vidjil-algo-${release_name} next"
173

174
175
.testing-compilers: &test_various_compilers
  stage: multiple_tests
176
177
178
179
180
181
182
183
184
185
186
187
  tags:
    - several-compilers
  script:
    - |
      for CXX in $COMPILERS; do
        echo "**** Using $CXX for compilation ****"
        make -C algo clean
        make -C algo CXX=$CXX unit
        make -C algo CXX=$CXX should
        make -C algo CXX=$CXX valgrind_unit
      done

188
189
190
191
192
test_various_compilers_manual:
  <<: *test_various_compilers
  when: manual
  only:
    - /^feature-.*a.*\/.*$/
193
    - schedules
194
195
196
197
198
199

test_various_compilers_release:
  <<: *test_various_compilers
  only:
    - /^feature-.*a.*\/release$/

200

Mathieu Giraud's avatar
Mathieu Giraud committed
201
# Client
202

203
test_browser_unit:
204
  stage: test_unit
Mathieu Giraud's avatar
Mathieu Giraud committed
205
  script: make unit_browser
206
207
208
  artifacts:
    paths:
    - browser/
209
  only:
210
211
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
212
    - prod-client
213
    - dev
214
    - schedules
215
216
  tags:
    - web
217

218
219
test_browser-functional:
  stage: test_functional
220
  retry: 2
221
222
223
224
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
225
226
227
228
229
    - HEADLESS=1 make -C browser/test functional
  artifacts:
    paths:
    - browser/
  only:
230
231
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
232
    - prod-client
233
    - schedules
234
235
  tags:
    - web
236
237
238

test_browser-functional-external:
  stage: test_functional_external
239
  retry: 2
240
241
242
243
244
  script:
    - make -C browser
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
    - HEADLESS=1 make -C browser/test external
245
246
247
  artifacts:
    paths:
    - browser/
248
  only:
249
250
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
Mathieu Giraud's avatar
Mathieu Giraud committed
251
    - prod-client
252
    - schedules
253
254
  tags:
    - web
255

256
257
code_quality:
  stage: test_quality
258
  script: make -C browser quality
259
  only:
260
261
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
262
    - prod-client
263
    - dev
264
    - schedules
265
266
  tags:
    - web
267

Ryan Herbert's avatar
Ryan Herbert committed
268
269
270
271
272
# Server

test_server_unit:
    stage: test_unit
    script:
273
274
      - virtualenv $CI_BUILD_REF_SLUG
      - source $CI_BUILD_REF_SLUG/bin/activate
275
      - pip install -r requirements.txt
Ryan Herbert's avatar
Ryan Herbert committed
276
      - make -C server install_web2py_standalone
277
      - make -C server launch_fuse_server
278
      - make unit_server || (make -C server kill_fuse_server; deactivate; false)
Ryan Herbert's avatar
Ryan Herbert committed
279
      - make -C server kill_fuse_server
280
      - deactivate
Ryan Herbert's avatar
Ryan Herbert committed
281
    only:
282
283
      - /^feature-.*s.*\/.*$/
      - /^hotfix-.*s.*\/.*$/
Ryan Herbert's avatar
Ryan Herbert committed
284
285
      - prod-server
      - dev
286
      - schedules
Ryan Herbert's avatar
Ryan Herbert committed
287
288
289
    tags:
      - web

290
291
292
test_server_functional:
  stage: test_functional
  script:
293
294
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
295
    - cd docker && docker-compose up -d && cd ..
Ryan Herbert's avatar
Ryan Herbert committed
296
    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
297
    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
298
    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
299
300
    - source /etc/profile.d/rvm.sh
    - rvm use 2.1.1
301
    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; false)
302
    - cd docker && docker-compose stop
303
304
305
306
307
  only:
    - /^feature-.*s.*\/.*$/
    - /^hotfix-.*s.*\/.*$/
    - prod-server
    - dev
308
    - schedules
309
  tags:
310
    - x86_64
311
    - docker
312

Ryan Herbert's avatar
Ryan Herbert committed
313
314
# Deployment

315
316
deploy_review:
  stage: deploy_review
317
  script:
318
    - make -C browser icons
319
    - echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
320
    - rsync -av --delete browser $DEPLOY_SERVER:$DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG
321
  environment:
322
    name: review/$CI_BUILD_REF_NAME
Mikaël Salson's avatar
Mikaël Salson committed
323
    url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
Mathieu Giraud's avatar
Mathieu Giraud committed
324
    on_stop: stop_deploy_review
325
  only:
326
327
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
328
329
  tags:
    - web
330

331
332
333
334
335
336
337
338
339
340
deploy_docker:
  stage: deploy_docker
  script:
    - docker tag $CI_BUILD_REF_SLUG vidjil/vidjil:latest
    - docker push vidjil/vidjil:latest
  when: manual
  only:
    - prod-server
  tags:
    - web
341
    - docker
342

343
deploy_prod:
344
  stage: deploy_prod
345
  script:
346
347
348
349
350
351
352
    - ssh $PROD_CLIENT_SERVER "
         cd $PROD_CLIENT_PATH;
         git fetch
         && git reset --hard origin/prod-client
         && make -C browser
         && make -C browser sha1
         && cp doc/analysis-example.vidjil browser/"
353
354
  environment:
    name: production
355
    url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
356
357
  only:
    - prod-client
358
359
  tags:
    - web
360

361
362
stop_deploy_review:
  stage: deploy_review
363
364
365
366
367
368
  variables:
    GIT_STRATEGY: none
  script:
    - ssh $DEPLOY_SERVER "rm -rf $DEPLOY_SERVER_PATH$CI_BUILD_REF_SLUG"
  when: manual
  environment:
369
    name: review/$CI_BUILD_REF_NAME
370
    action: stop
371
  only:
372
373
    - /^feature-.*c.*\/.*$/
    - /^hotfix-.*c.*\/.*$/
374
  tags:
375
    - web