Commit f3a88137 authored by flothoni's avatar flothoni
Browse files

Merge branch 'dev' of gitlab.inria.fr:vidjil/vidjil into feature-c/4370-label-tooltip-graph

parents ea613252 4aabe687
Pipeline #192740 passed with stages
in 19 minutes and 40 seconds
......@@ -51,12 +51,13 @@ test_germlines:
- /^feature-.*g.*\/.*$/
- schedules
# Doc
# Includes
include:
- local: '/doc/.gitlab-ci.yml'
- local: '/doc/.gitlab-ci.yml' # Doc
- local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
- local: 'algo/.gitlab-ci.yml' # Vidjil-algo pipelines
- template: Code-Quality.gitlab-ci.yml # Code quality
# Algorithm
......@@ -127,6 +128,7 @@ algo_valgrind_functional:
release_valgrind_functional:
<<: *valgrind_functional
allow_failure: true
only:
- /^feature-.*a.*\/release$/
......@@ -187,6 +189,7 @@ test_browser_unit:
artifacts:
paths:
- browser/
when: always
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
......@@ -206,7 +209,8 @@ test_browser_unit:
- HEADLESS=1 make -C browser/test functional
artifacts:
paths:
- browser/
- browser/
when: always
reports:
junit: browser/test/test/TEST-*.xml
only:
......@@ -233,6 +237,7 @@ test_browser_unit:
artifacts:
paths:
- browser/
when: always
reports:
junit: browser/test/test/TEST-*.xml
only:
......@@ -284,7 +289,14 @@ old-chrome-browser-functional-external-chrome:
tags:
- legacy
# Code quality
code_quality:
stage: test_quality
artifacts:
paths: [gl-code-quality-report.json]
code_quality_custom:
stage: test_quality
script: make -C browser quality
only:
......@@ -301,8 +313,8 @@ code_quality:
test_server_unit:
stage: test_unit
script:
- virtualenv $CI_BUILD_REF_SLUG
- source $CI_BUILD_REF_SLUG/bin/activate
- virtualenv $CI_COMMIT_SHORT_SHA
- source $CI_COMMIT_SHORT_SHA/bin/activate
- pip install -r requirements.txt
- make -C server install_web2py_standalone
- make -C server launch_fuse_server
......@@ -411,6 +423,7 @@ deploy_review_server:
- make -C browser icons
- cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
- sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
- 'sed -i "s/\"server_id\" : \"\"/\"server_id\" : \"${CI_BUILD_REF_SLUG}.server.ci.vidjil.org\"/" ./browser/js/conf.js'
- sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
- sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py
- mv docker docker_${CI_BUILD_REF_SLUG}
......@@ -422,7 +435,7 @@ deploy_review_server:
- sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml
- cd ..
- rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac)
- ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers"
- ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers; sleep 20s; docker-compose exec -T uwsgi bash -c 'cd /usr/share/vidjil/server/web2py/applications/vidjil/scripts; /usr/local/bin/gosu 1005 bash launch_python_script.sh load-sql.py -n /usr/share/vidjil/docker/ci/ci.sql'"
environment:
name: review_server/$CI_BUILD_REF_NAME
url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/
......@@ -457,12 +470,7 @@ deploy_prod:
stage: deploy_prod
script:
- ssh $PROD_CLIENT_SERVER "
cd $PROD_CLIENT_PATH;
git fetch
&& git reset --hard origin/prod-client
&& make -C browser
&& make -C browser sha1
&& cp doc/analysis-example.vidjil browser/"
cd $PROD_CLIENT_PATH;"
environment:
name: production
url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
......
## Preparation
## Before the release
### Documentation
* [ ] New features are described in `doc/vidjil-algo.md`
* [ ] Breaking changes (or needed) are understood
* [ ] Breaking changes (and needed configuration changes) are understood
On the `feature-a/release` branch, the last commit is the release commit updating the three following files:
* [ ] CHANGELOG
* [ ] `doc/vidjil-algo.md` with the proper release tag
* [ ] `algo/release` with the proper release tag
## Pipelines
### Pipelines
https://gitlab.inria.fr/vidjil/vidjil/pipelines/XXXXX
(if tests passed on different pipelines, indicate below the relevant pipelines)
Usual tests should pass, but also:
* [ ] prepare_release
......@@ -23,21 +26,23 @@ Benchmarks
* [ ] almost no change...
* [ ] ... or significant changes are understood
## Tag and push
## The release, publish, tag and push
* [ ] merge this MR to *master* and tag: `git tag release-20XX-XX`
* [ ] push: `git push origin master release-20XX-XX`
* [ ] mirror to GH: `git push github master release-20XX.XX`
* [ ] click `publish_release` (there may be changes to CD...)
* [ ] deploy the doc
After these steps, we merge back the release in `dev`:
* [ ] merge this MR to dev
* [ ] tag (`git tag release-20XX-XX; git push origin release-20XX-XX`)
* [ ] mirror to GH
## After the release: deploy
## Deploy
* [ ] Click `deploy_release_prod` in the `deploy_prod` stage, deploying the release on `vidjil-algo-next` on the public server
* [ ] Open a new internal issue with the `Deploy (vidjil-algo)` template : vdj#XXXXX
* [ ] doc
* [ ] `app` (vidjil-algo-next)
* [ ] `app` if needed, update server configs
* [ ] `app` notification to users
* [ ] `app` (vidjil-algo)
* [ ] `app` test on production (X5 and L4)
* [ ] `hds` queue for qualification ?
* [ ] twice a year, communicate to users
/label ~cpp
......@@ -125,6 +125,16 @@ test_gcc9_release:
extends:
- .testing-gcc
- .testing_various_compilers_release
test_gcc10_manual:
image: gcc:10
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc10_release:
image: gcc:10
extends:
- .testing-gcc
- .testing_various_compilers_release
test_clang34_manual:
image: debian:8-slim
variables:
......
#ifndef TOOLS_H
#define TOOLS_H
#include <string>
using namespace std ;
typedef string junction ;
......
!NO_LAUNCHER:
!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
!REQUIRES: which valgrind
!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
$ No invalid read with short sequences
e1:ERROR SUMMARY: 0 errors from 0 contexts
......
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
!OUTPUT_FILE: out/bug4225-j.affects
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
$ Find only +k and ? affects before the stretch of _ for all loci
16: seed .*(\+k| \?){28}( _)+$
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
!OUTPUT_FILE: out/chimera-fake-half.affects
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
$ Find only +B and ? affects on the TRB and unexpected lines
2: seed .* _(\+B| \?){48} _
!REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/fused.data ; cat out/fused.data | python $VIDJIL_DIR/tools/format_json.py -1
!OUTPUT_FILE: out/algo-then-fuse.vidjil
!OPTIONS: --mod jR
!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/algo-then-fuse.vidjil
$ Points list
1:"original_names": \[".*data//Stanford_S22.fasta", ".*data//Stanford_S22.fasta"\]
2:samples.original_names
:samples.original_names[0]: data//Stanford_S22.fasta
:samples.original_names[1]: data//Stanford_S22.fasta
$ Most abundant window, twice, fused
1:"id": "CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC".*"reads": \[8, 8\].*"top": 2
$ Most abundant window, twice, fused
:clones[0].id: CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC
:clones[0].reads: [8, 8]
:clones[0].top: 1
$ Windows that are not in the bottom of the distribution
1: "distribution": \{[^}]+ "0.00001": \[9062, 9062\]
# :reads.distribution.0.00001: should#59
:reads.distribution: [9062, 9062]
\ No newline at end of file
!REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py -o out/S22.fasta out/Stanford_S22.vidjil ;
!OUTPUT_FILE: out/S22.fasta
!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py -o out/S22.fasta out/Stanford_S22.vidjil ;
$ 5 representative sequences in the FASTA output file
5:>
!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
!EXIT_CODE: 1
!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
$Check that correct custom cost is used
1:use custom Cost "1, 2, 3, 4, 5"
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
!OUTPUT_FILE: out/cdr3-stopcodon.vidjil
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
$ Two identical junctions in JSON
2: "CATWDRKNYYKKLF"
......
!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 --out-vdjfa -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
# Testing -x/-y/-z options
......
!LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG --consensus-on-longest-sequences $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-longest.log
!LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-random.log
!NO_LAUNCHER:
!LAUNCH: diff consensus-longest.log consensus-random.log
!EXIT_CODE: 1
!LAUNCH: diff consensus-longest.log consensus-random.log
$ Output should differ: ReadQualityScore gives a consensus of 52bp (with the spurious insertion)
# Appears twice in the header of the consensus sequence and in the similarity matrix
......
!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
!EXIT_CODE: 1
!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
$ Deprecated options
5:is deprecated
......
!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
!EXIT_CODE: 109
!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
$ Unknown option
1:error.* --hello
......
!NO_LAUNCHER:
!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
!EXIT_CODE: 1
!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
$ Error, no germlines
1:error.* one germline must be given
......
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
!EXIT_CODE: 1
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
$ Error, incorrect *.g
1:error.* cannot properly read
......
......@@ -13,9 +13,8 @@ $ Segment two reads on IgVC/IgJC (second -g option)
1: IgVC .* -> .* 1
1: IgJC .* -> .* 1
!LAUNCH: cat out/isotypes.vidjil
!NO_LAUNCHER:
!LAUNCH: cat out/isotypes.vidjil
$ Both germlines are in .vidjil
1: /germline"
......
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a $VIDJIL_DATA/clones_simul.fa
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a --out-vdjfa $VIDJIL_DATA/clones_simul.fa
$ Output
1: out-a.vidjil
1: out-a.tsv
1: out-a.vdj.fa
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b $VIDJIL_DATA/clones_simul.fa
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b --out-vdjfa $VIDJIL_DATA/clones_simul.fa
$ Compressed output
1: out-b.vidjil.gz
......
!NO_LAUNCHER:
cd $VIDJIL_DIR
./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' | sh
cd $VIDJIL_DIR ; ./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' > help-examples.sh
cd $VIDJIL_DIR ; sh help-examples.sh
# Test examples embedded in './vidjil-algo -h'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment