Merge branch 'dev' of gitlab.inria.fr:vidjil/vidjil into feature-c/4370-label-tooltip-graph

.gitlab-ci.yml

@@ -51,12 +51,13 @@ test_germlines:
     - /^feature-.*g.*\/.*$/
     - schedules
 
-# Doc
+# Includes
 
 include:
-  - local: '/doc/.gitlab-ci.yml'
+  - local: '/doc/.gitlab-ci.yml' # Doc
   - local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
   - local: 'algo/.gitlab-ci.yml' # Vidjil-algo pipelines
+  - template: Code-Quality.gitlab-ci.yml # Code quality
 
 # Algorithm
 
@@ -127,6 +128,7 @@ algo_valgrind_functional:
 
 release_valgrind_functional:
   <<: *valgrind_functional
+  allow_failure: true
   only:
     - /^feature-.*a.*\/release$/
 
@@ -187,6 +189,7 @@ test_browser_unit:
   artifacts:
     paths:
     - browser/
+    when: always
   only:
     - /^feature-.*c.*\/.*$/
     - /^hotfix-.*c.*\/.*$/
@@ -206,7 +209,8 @@ test_browser_unit:
     - HEADLESS=1 make -C browser/test functional
   artifacts:
     paths:
-    - browser/
+    - browser/ 
+    when: always
     reports:
       junit: browser/test/test/TEST-*.xml
   only:
@@ -233,6 +237,7 @@ test_browser_unit:
   artifacts:
     paths:
     - browser/
+    when: always
     reports:
       junit: browser/test/test/TEST-*.xml
   only:
@@ -284,7 +289,14 @@ old-chrome-browser-functional-external-chrome:
   tags:
     - legacy
 
+# Code quality
+
 code_quality:
+  stage: test_quality
+  artifacts:
+    paths: [gl-code-quality-report.json]
+
+code_quality_custom:
   stage: test_quality
   script: make -C browser quality
   only:
@@ -301,8 +313,8 @@ code_quality:
 test_server_unit:
     stage: test_unit
     script:
-      - virtualenv $CI_BUILD_REF_SLUG
-      - source $CI_BUILD_REF_SLUG/bin/activate
+      - virtualenv $CI_COMMIT_SHORT_SHA
+      - source $CI_COMMIT_SHORT_SHA/bin/activate
       - pip install -r requirements.txt
       - make -C server install_web2py_standalone
       - make -C server launch_fuse_server
@@ -411,6 +423,7 @@ deploy_review_server:
     - make -C browser icons
     - cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
     - sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
+    - 'sed -i "s/\"server_id\" : \"\"/\"server_id\" : \"${CI_BUILD_REF_SLUG}.server.ci.vidjil.org\"/" ./browser/js/conf.js'
     - sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
     - sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py
     - mv docker docker_${CI_BUILD_REF_SLUG}
@@ -422,7 +435,7 @@ deploy_review_server:
     - sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml
     - cd ..
     - rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac)
-    - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers"
+    - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers; sleep 20s; docker-compose exec -T uwsgi bash -c 'cd /usr/share/vidjil/server/web2py/applications/vidjil/scripts; /usr/local/bin/gosu 1005 bash launch_python_script.sh load-sql.py -n /usr/share/vidjil/docker/ci/ci.sql'"
   environment:
     name: review_server/$CI_BUILD_REF_NAME
     url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/
@@ -457,12 +470,7 @@ deploy_prod:
   stage: deploy_prod
   script:
     - ssh $PROD_CLIENT_SERVER "
-         cd $PROD_CLIENT_PATH;
-         git fetch
-         && git reset --hard origin/prod-client
-         && make -C browser
-         && make -C browser sha1
-         && cp doc/analysis-example.vidjil browser/"
+         cd $PROD_CLIENT_PATH;"
   environment:
     name: production
     url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil

.gitlab/merge_request_templates/Release (vidjil-algo).md

 
-## Preparation
+## Before the release
+
+### Documentation
 
 * [ ] New features are described in `doc/vidjil-algo.md`
-* [ ] Breaking changes (or needed) are understood
+* [ ] Breaking changes (and needed configuration changes) are understood
  
 On the `feature-a/release` branch, the last commit is the release commit updating the three following files:
  * [ ] CHANGELOG
  * [ ] `doc/vidjil-algo.md` with the proper release tag
  * [ ] `algo/release` with the proper release tag
 
-## Pipelines
+### Pipelines
 
 https://gitlab.inria.fr/vidjil/vidjil/pipelines/XXXXX
+(if tests passed on different pipelines, indicate below the relevant pipelines)
 
 Usual tests should pass, but also:
 * [ ] prepare_release
@@ -23,21 +26,23 @@ Benchmarks
 * [ ] almost no change...
 * [ ] ... or significant changes are understood
 
-## Tag and push
+## The release, publish, tag and push
+
+* [ ] merge this MR to *master* and tag: `git tag release-20XX-XX` 
+* [ ] push: `git push origin master release-20XX-XX`
+* [ ] mirror to GH: `git push github master release-20XX.XX`
+* [ ] click `publish_release` (there may be changes to CD...)
+* [ ] deploy the doc
+
+
+After these steps, we merge back the release in `dev`:
+* [ ] merge this MR to dev
 
-* [ ] tag (`git tag release-20XX-XX; git push origin release-20XX-XX`)
-* [ ] mirror to GH
+## After the release: deploy
 
-## Deploy
+* [ ] Click `deploy_release_prod` in the `deploy_prod` stage, deploying the release on `vidjil-algo-next` on the public server
+* [ ] Open a new internal issue with the `Deploy (vidjil-algo)` template : vdj#XXXXX
 
-* [ ] doc 
-* [ ] `app` (vidjil-algo-next)
-* [ ] `app` if needed, update server configs
-* [ ] `app` notification to users
-* [ ] `app` (vidjil-algo)
-* [ ] `app` test on production (X5 and L4)
-* [ ] `hds` queue for qualification ?
-* [ ] twice a year, communicate to users
 
 /label ~cpp
 

algo/.gitlab-ci-compilers.yml

@@ -125,6 +125,16 @@ test_gcc9_release:
   extends:
     - .testing-gcc
     - .testing_various_compilers_release
+test_gcc10_manual:
+  image: gcc:10
+  extends:
+    - .testing-gcc
+    - .testing_various_compilers_manual
+test_gcc10_release:
+  image: gcc:10
+  extends:
+    - .testing-gcc
+    - .testing_various_compilers_release
 test_clang34_manual:
   image: debian:8-slim
   variables:

algo/core/tools.h

 #ifndef TOOLS_H
 #define TOOLS_H
+#include <string>
 
 using namespace std ;
 typedef string junction ;

algo/release

-2020.06
+2020.08

algo/tests/bugs/bug20141024.should

 !NO_LAUNCHER:
-!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full  $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
 !REQUIRES: which valgrind
+!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full  $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
+
 
 $ No invalid read with short sequences
 e1:ERROR SUMMARY: 0 errors from 0 contexts

algo/tests/bugs/bug4225.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
 !OUTPUT_FILE: out/bug4225-j.affects
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
 
 $ Find only +k and ? affects before the stretch of _ for all loci
 16: seed .*(\+k| \?){28}( _)+$
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
 !OUTPUT_FILE: out/chimera-fake-half.affects
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
 
 $ Find only +B and ? affects on the TRB and unexpected lines
 2: seed .* _(\+B| \?){48} _

algo/tests/should-get-tests/algo-then-fuse.should

 !REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
-!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/fused.data ; cat out/fused.data | python $VIDJIL_DIR/tools/format_json.py -1
+
+!OUTPUT_FILE: out/algo-then-fuse.vidjil
+!OPTIONS: --mod jR
+
+!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/algo-then-fuse.vidjil
 
 $ Points list
-1:"original_names": \[".*data//Stanford_S22.fasta", ".*data//Stanford_S22.fasta"\]
+2:samples.original_names
+:samples.original_names[0]: data//Stanford_S22.fasta
+:samples.original_names[1]: data//Stanford_S22.fasta
 
-$ Most abundant window, twice, fused 
-1:"id": "CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC".*"reads": \[8, 8\].*"top": 2
+$ Most abundant window, twice, fused
+:clones[0].id: CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC
+:clones[0].reads: [8, 8]
+:clones[0].top: 1
 
 $ Windows that are not in the bottom of the distribution
-1: "distribution": \{[^}]+ "0.00001": \[9062, 9062\]
+#  :reads.distribution.0.00001:  should#59
+:reads.distribution: [9062, 9062]
\ No newline at end of file

algo/tests/should-get-tests/algo-then-vidjil-to-fasta.should

 !REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
-!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py  -o out/S22.fasta out/Stanford_S22.vidjil ;
 !OUTPUT_FILE: out/S22.fasta
+!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py  -o out/S22.fasta out/Stanford_S22.vidjil ;
+
 
 $ 5 representative sequences in the FASTA output file
 5:>

algo/tests/should-get-tests/analysis-cost.should

-!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
+
 
 $Check that correct custom cost is used
 1:use custom Cost "1, 2, 3, 4, 5"

algo/tests/should-get-tests/cdr3-stopcodon.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
 !OUTPUT_FILE: out/cdr3-stopcodon.vidjil
+!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
 
 $ Two identical junctions in JSON
 2: "CATWDRKNYYKKLF"

algo/tests/should-get-tests/config-write-then-read.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 --out-vdjfa -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
 
 # Testing -x/-y/-z options
 

algo/tests/should-get-tests/consensus-random-read-chooser.should

 !LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG --consensus-on-longest-sequences $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-longest.log
 !LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-random.log
 !NO_LAUNCHER:
-!LAUNCH: diff consensus-longest.log consensus-random.log
 !EXIT_CODE: 1
+!LAUNCH: diff consensus-longest.log consensus-random.log
 
 $ Output should differ: ReadQualityScore gives a consensus of 52bp (with the spurious insertion)
 # Appears twice in the header of the consensus sequence and in the similarity matrix

algo/tests/should-get-tests/error-deprecated-options.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
+
 
 $ Deprecated options
 5:is deprecated

algo/tests/should-get-tests/error-unknown-option.should

-!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
 !EXIT_CODE: 109
+!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
+
 
 $ Unknown option
 1:error.* --hello

algo/tests/should-get-tests/germlines-error-no-germline.should

 !NO_LAUNCHER:
-!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 
 $ Error, no germlines
 1:error.* one germline must be given

algo/tests/should-get-tests/germlines-error.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
+
 
 $ Error, incorrect *.g
 1:error.* cannot properly read

algo/tests/should-get-tests/germlines-several.should

@@ -13,9 +13,8 @@ $ Segment two reads on IgVC/IgJC (second -g option)
 1: IgVC .* -> .* 1
 1: IgJC .* -> .* 1 
 
-
-!LAUNCH: cat out/isotypes.vidjil
 !NO_LAUNCHER:
+!LAUNCH: cat out/isotypes.vidjil
 
 $ Both germlines are in .vidjil
 1: /germline"

algo/tests/should-get-tests/gzip-out.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a $VIDJIL_DATA/clones_simul.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a --out-vdjfa $VIDJIL_DATA/clones_simul.fa
 
 $ Output
 1: out-a.vidjil
 1: out-a.tsv
 1: out-a.vdj.fa
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b $VIDJIL_DATA/clones_simul.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b --out-vdjfa $VIDJIL_DATA/clones_simul.fa
 
 $ Compressed output
 1: out-b.vidjil.gz

algo/tests/should-get-tests/help-examples.should

 !NO_LAUNCHER:
 
-cd $VIDJIL_DIR
-./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' | sh
+cd $VIDJIL_DIR ; ./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' > help-examples.sh
+cd $VIDJIL_DIR ; sh help-examples.sh
 
 # Test examples embedded in './vidjil-algo -h'