Commit b2bb473e authored by Thonier Florian's avatar Thonier Florian
Browse files

Merge branch 'dev' of gitlab.inria.fr:vidjil/vidjil into feature-s/3154-pre_process_json

parents 81028d71 771eef07
Pipeline #286638 failed with stages
in 23 minutes and 1 second
......@@ -29,7 +29,6 @@ dep.mk
/browser/cgi/align.cgi
/browser/js/conf.js
node_modules/
cypress/
server/web2py/applications/vidjil/errors
*.bak
......@@ -50,3 +49,9 @@ germline/IMGT_RELEASE
vidjil-algo
server/web2py/gluon/packages/dal/
cypress/reports
cypress/screenshots
cypress/videos
docker/volumes/
......@@ -13,10 +13,17 @@ stages:
- deploy_review
- test_germlines
- test_functional
- test_cypress_client_supported
- test_cypress_client_latest
- test_cypress_client_legacy
- test_cypress_server_supported
- test_cypress_server_latest
- test_cypress_server_legacy
- test_tutorial
- test_tutorial_server
- test_shouldvdj
- test_functional_external
- coverage
- prepare_release
- tgz_release
- valgrind_unit
- valgrind_functional
......@@ -24,7 +31,6 @@ stages:
- benchmark
- publish_release
- deploy_prod
# Anchors
.coverage_dependency:
......@@ -49,7 +55,7 @@ test_germlines:
- make -C germline tests
only:
- /^feature-.*g.*\/.*$/
- schedules
# Includes
......@@ -57,6 +63,8 @@ include:
- local: '/doc/.gitlab-ci.yml' # Doc
- local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
- local: 'algo/.gitlab-ci.yml' # Vidjil-algo pipelines
- local: 'browser/.gitlab-ci.yml' # Client pipelines
- local: 'server/.gitlab-ci.yml' # Server pipelines
- template: Code-Quality.gitlab-ci.yml # Code quality
# Algorithm
......@@ -122,6 +130,7 @@ algo_valgrind_unit:
algo_valgrind_functional:
<<: *valgrind_functional
when: manual
dependencies: []
only:
- /^feature-.*a.*\/.*$/
- schedules
......@@ -132,13 +141,6 @@ release_valgrind_functional:
only:
- /^feature-.*a.*\/release$/
prepare_release:
stage: prepare_release
script: make -C algo release RELEASE_TAG='rc'
when: manual
only:
- /^feature-.*a.*\/.*$/
tgz_release:
stage: tgz_release
script: make -C algo release RELEASE_TAG=`cat algo/release`
......@@ -162,7 +164,7 @@ copy_release:
scp algo/releases/vidjil-algo*.tar.gz algo/releases/vidjil-algo*/vidjil-algo-*_* $repo
done
release_name=$(cat algo/release)
ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-latest_x86_64"
ssh $VIDJIL_WWW "cd /var/www/html/releases; ln -sf vidjil-algo-${release_name}.tar.gz vidjil-algo-latest.tgz; ln -sf vidjil-algo-${release_name}_x86_64 vidjil-algo-latest_x86_64"
ssh $VIDJIL_BONSAI "cd /bio1/www/html/vidjil; scp vidjil-algo-${release_name}* $VIDJIL_BONSAI_PROD:/bio1/www/html/vidjil"
when: manual
only:
......@@ -193,7 +195,7 @@ test_browser_unit:
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
- prod-client
- /^prod-client/
- dev
- schedules
tags:
......@@ -216,7 +218,7 @@ test_browser_unit:
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
- prod-client
- /^prod-client/
- schedules
tags:
- web
......@@ -243,7 +245,7 @@ test_browser_unit:
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
- prod-client
- /^prod-client/
- schedules
tags:
- web
......@@ -289,6 +291,84 @@ old-chrome-browser-functional-external-chrome:
tags:
- legacy
.browser-tutorial:
stage: test_tutorial
retry: 2
script:
- make -C browser
- source /etc/profile.d/rvm.sh
- rvm use 2.6.1
- HEADLESS=1 make test_tutorial
artifacts:
paths:
- browser/
reports:
junit: browser/test/test/TEST-*.xml
when: always
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
- /^prod-client/
- schedules
- /tutorial/
tags:
- web-pandoc
.browser-tutorial-chrome:
extends: .browser-tutorial
variables:
WATIR_CHROME: "1"
ff32-browser-tutorial:
extends: .browser-tutorial
variables:
FIREFOX_VERSION: 32
ff45-browser-tutorial:
extends: .browser-tutorial
variables:
FIREFOX_VERSION: 45
chrome-browser-tutorial:
extends: .browser-tutorial-chrome
old-chrome-browser-tutorial:
extends: .browser-tutorial-chrome
tags:
- legacy
server_tutorial_server:
stage: test_tutorial_server
extends: .test_server_base
retry: 1
script:
- source /etc/profile.d/rvm.sh
- rvm use 2.6.1
- HEADLESS=1 make test_tutorial_server
artifacts:
paths:
- browser/
- server/web2py/applications/vidjil/
reports:
junit: server/web2py/applications/vidjil/test/TEST-*.xml
when: always
only:
- /^feature-.*s.*\/.*$/
- /^hotfix-.*s.*\/.*$/
- prod-server
- dev
- schedules
- /tutorial/
tags:
- web-pandoc
variables:
FIREFOX_VERSION: 45
# Code quality
code_quality:
......@@ -302,7 +382,7 @@ code_quality_custom:
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
- prod-client
- /^prod-client/
- dev
- schedules
tags:
......@@ -327,15 +407,15 @@ test_server_unit:
only:
- /^feature-.*s.*\/.*$/
- /^hotfix-.*s.*\/.*$/
- prod-server
- /^prod-server/
- dev
- schedules
tags:
- web
.test_server_functional:
.test_server_base:
stage: test_functional
script:
before_script:
- docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
- docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
- sed -i '/\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
......@@ -350,25 +430,42 @@ test_server_unit:
- sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_tests.sh
- docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
- docker exec docker_nginx_1 make -C /usr/share/vidjil browser
- source /etc/profile.d/rvm.sh
- rvm use 2.6.1
- HEADLESS=1 make functional_server
after_script:
- cd docker && docker-compose down -v
- docker rmi "vidjil/server:test" "vidjil/client:test" reporter:test
artifacts:
reports:
junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml
when: always
expire_in: 1 week
paths:
- server/web2py/ap‎plications/vidjil‎/tests/
only:
- /^feature-.*s.*\/.*$/
- /^hotfix-.*s.*\/.*$/
- prod-server
- /^prod-server/
- dev
- schedules
tags:
- x86_64
- docker
.test_server_functional:
stage: test_functional
extends: .test_server_base
script:
- source /etc/profile.d/rvm.sh
- rvm use 2.6.1
- HEADLESS=1 make functional_server
artifacts:
reports:
junit: server/web2py/applications/vidjil/tests/test/TEST-*.xml
paths:
- browser/
- server/web2py/applications/vidjil/
when: always
ff32-server-functional:
extends: .test_server_functional
variables:
......@@ -383,7 +480,7 @@ ff45-server-functional:
# Benchmark
benchmark_algo:
.benchmark-algo:
image: gcc:6.3
extends: .install-algo-dependencies
stage: benchmark
......@@ -393,11 +490,22 @@ benchmark_algo:
paths:
- algo/tests/benchmark.log
when: manual
dependencies: []
only:
- /^feature-.*a.*\/.*$/
tags:
- cidocker
benchmark-algo:
extends: .benchmark-algo
script:
- cd algo/tests ; python3 benchmark-releases.py -r 3 -b -c -i 2018.10 -i 2019.03 -i 2020.04 -i 2021.02.2 -i alpha-2102
benchmark-algo-full:
extends: .benchmark-algo
script:
- cd algo/tests ; python3 benchmark-releases.py -r 3 -bIc
# Deployment
......@@ -407,6 +515,7 @@ deploy_review:
- make -C browser icons
- echo "$REVIEW_CONFIG_JS" > browser/js/conf.js
- rsync -av --delete browser $REVIEW_SERVER:$REVIEW_SERVER_PATH$CI_BUILD_REF_SLUG
dependencies: []
environment:
name: review/$CI_BUILD_REF_NAME
url: http://$CI_BUILD_REF_SLUG.ci.vidjil.org/?data=analysis-example.vidjil
......@@ -423,8 +532,12 @@ deploy_review_server:
- make -C browser icons
- cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
- sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
- 'sed -i "s/\"server_id\" : \"\"/\"server_id\" : \"${CI_BUILD_REF_SLUG}.server.ci.vidjil.org\"/" ./browser/js/conf.js'
- sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
- sed -i -e "/EMAIL_SUBJECT_START/c EMAIL_SUBJECT_START='[Vidjil/CI][$CI_COMMIT_REF_NAME]'" -e '/FROM_EMAIL/c FROM_EMAIL="server.ci@vidjil.org"' ./server/web2py/applications/vidjil/modules/defs.py
- sed -ri 's/^(uwsgi_pass )\$upstream_(.*)$/\1\2/g' docker/vidjil-client/conf/uwsgi.conf
- mkdir browser/cgi
- cp docker/vidjil-client/conf/*.cgi browser/cgi
- mv docker docker_${CI_BUILD_REF_SLUG}
- cd docker_${CI_BUILD_REF_SLUG}
- TMP_DOCKER=$(tempfile)
......@@ -437,9 +550,10 @@ deploy_review_server:
- ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; mkdir -p postfix/ssl; docker-compose up -d mysql fuse uwsgi nginx workers; sleep 20s; docker-compose exec -T uwsgi bash -c 'cd /usr/share/vidjil/server/web2py/applications/vidjil/scripts; /usr/local/bin/gosu 1005 bash launch_python_script.sh load-sql.py -n /usr/share/vidjil/docker/ci/ci.sql'"
environment:
name: review_server/$CI_BUILD_REF_NAME
url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/
url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/
on_stop: stop_deploy_review_server
auto_stop_in: 2 months
dependencies: []
only:
- /^feature-.*s.*\/.*$/
- /^hotfix-.*s.*\/.*$/
......@@ -459,8 +573,8 @@ deploy_docker:
- docker rmi $TAG "vidjil/$TYPE:latest"
when: manual
only:
- prod-server
- prod-client
- /^prod-server/
- /^prod-client/
tags:
- web
- docker
......@@ -474,7 +588,7 @@ deploy_prod:
name: production
url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil
only:
- prod-client
- /^prod-client/
tags:
- web
......@@ -502,6 +616,7 @@ stop_deploy_review:
environment:
name: review/$CI_BUILD_REF_NAME
action: stop
dependencies: []
only:
- /^feature-.*c.*\/.*$/
- /^hotfix-.*c.*\/.*$/
......@@ -513,9 +628,11 @@ stop_deploy_review_server:
script:
- ssh $REVIEW_SERVER "set -o xtrace; docker-compose -f $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG/docker_$CI_BUILD_REF_SLUG/docker-compose.yml down -v ; rm -rf $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG; $REVIEW_SERVER_PATH_SERVER/uninstall_certs.sh $CI_BUILD_REF_SLUG"
when: manual
dependencies: []
environment:
name: review_server/$CI_BUILD_REF_NAME
action: stop
only:
- /^feature-.*s.*\/.*$/
- /^hotfix-.*s.*\/.*$/
### Before the deploy
- [ ] Set the deploy day (Most servers are busy on Mondays and Tuesdays. Web deploys are usually done on Wednesday.)
- [ ] Before the MR freeze, discuss regularly in the team what remains to do
Tuesday XX (D-8)
- [ ] Notification on the server
- [ ] MR Freeze (everything except urgent fixes or bikeshed strings/~doc)
- [ ] Move issues/MR that are still opened to a new milestone for the next release
- [ ] Push on `feature-cs/release`
### Deploy week
- [ ] Deploy on `dev`, either with `git pull` or with Docker
- [ ] Live tests on `dev`
On `feature-cs/release`:
- [ ] Bikeshed strings
- [ ] List all the changes, checking that the milestone contains all relevant issues/MR
- [ ] Finalize `CHANGELOG.web`
- [ ] Finalize ~doc
- [ ] Finalize `docker/CHANGELOG`
Tuesday XX (D-1)
- [ ] MR Freeze (everything)
### Deploy day
Wednesday XX (D0)
- [ ] Morning: update notification on the server
- [ ] MR `feature-cs/release` to `prod-client` !XXX and `prod-server` !XXX
- [ ] Disconnect the client through `conf.js`
- [ ] Deploy
- [ ] Link vidjil-algo to latest if needed
- [ ] Live test on the prod server
- [ ] Reconnect the client
- [ ] Notification "Maintenance finished" + news
- [ ] Possibly mail to users
- [ ] Merge `feature-cs/release` into `dev` and `master`, and `git push github`
......@@ -17,7 +17,6 @@ https://gitlab.inria.fr/vidjil/vidjil/pipelines/XXXXX
(if tests passed on different pipelines, indicate below the relevant pipelines)
Usual tests should pass, but also:
* [ ] prepare_release
* [ ] valgrind_unit
* [ ] valgrind_functional
* [ ] multiple_tests
......@@ -26,13 +25,19 @@ Benchmarks
* [ ] almost no change...
* [ ] ... or significant changes are understood
Profiling
* cpu: PointerACAutomaton.getResults, ~37% ?
* mem: PointerACAutomaton.insert, ~73% ?
* [ ] almost no change...
* [ ] ... or significant changes are understood
## The release, publish, tag and push
* [ ] merge this MR to *master* and tag: `git tag release-20XX-XX`
* [ ] push: `git push origin master release-20XX-XX`
* [ ] mirror to GH: `git push github master release-20XX.XX`
* [ ] click `publish_release` (there may be changes to CD...)
* [ ] deploy the doc
* [ ] deploy to vidjil.org/releases and app.vidjil.org: click on `copy_release` (there may be changes to CD...)
* [ ] deploy the doc: click on `deploy_doc`
After these steps, we merge back the release in `dev`:
......@@ -43,6 +48,7 @@ After these steps, we merge back the release in `dev`:
* [ ] Click `deploy_release_prod` in the `deploy_prod` stage, deploying the release on `vidjil-algo-next` on the public server
* [ ] Open a new internal issue with the `Deploy (vidjil-algo)` template : vdj#XXXXX
* [ ] Possibly fix things in `merge_request_templates/Release (vidjil-algo).md` on a new MR
/label ~cpp
......@@ -18,6 +18,16 @@ test_tools_if_python:
test_tools:
$(MAKE) -C tools/tests
test_tutorial:
$(MAKE) -C doc/tutorial tutorial-test.rb
$(MAKE) -C browser/test tutorial
test_tutorial_server:
$(MAKE) -C doc/tutorial tutorial-server.rb
wget http://vidjil.org/seqs/tutorial_dataset.zip
unzip -o tutorial_dataset.zip
$(MAKE) -C server tutorial
shouldvdj_generate:
@echo
rm -rf data/gen
......@@ -45,6 +55,47 @@ functional_server:
headless_server:
$(MAKE) -C server headless
###############################
### Browser tests WITH CYPRESS
build_cypress_image:
docker build ./docker/ci -t "vidjilci/cypress_with_browsers:latest"
functional_browser_cypress_open:
# Need to create a symbolic link; but allow to directly see result
# Usefull for fast debugging; allow to launch script one by one
cypress open --env workdir=$(PWD),host=local
functional_browser_cypress:
docker run \
-v $(PWD)/browser/test/cypress:/app/cypress \
-v $(PWD)/browser/test/data/:/app/cypress/fixtures/data/ \
-v $(PWD)/doc/:/app/cypress/fixtures/doc/ \
-v $(PWD):/app/vidjil \
-v "$(PWD)/docker/ci/cypress_script.bash":"/app/script.bash" \
-v "$(PWD)/docker/ci/cypress.json":"/app/cypress.json" \
--env BROWSER=electron --env HOST=local "vidjilci/cypress_with_browsers:latest" bash script.bash
functional_server_cypress_open:
cypress open --env workdir=$(PWD),host=local --project ./server/web2py/applications/vidjil/tests
functional_server_cypress:
# Need to have a local server deploy with the ci data integrated
docker run \
-v $(PWD)/server/web2py/applications/vidjil/tests/cypress:/app/cypress \
-v $(PWD)/browser/test/data/:/app/cypress/fixtures/data/ \
-v $(PWD)/doc/:/app/cypress/fixtures/doc/ \
-v $(PWD):/app/vidjil \
-v "$(PWD)/docker/ci/cypress_script.bash":"/app/script.bash" \
-v "$(PWD)/docker/ci/cypress.json":"/app/cypress.json" \
--network="host" \
--env BROWSER=electron --env HOST=local "vidjilci/cypress_with_browsers:latest" bash script.bash
###############################
tutorial-test.rb:
$(MAKE) -C doc/tutorial tutorial-test.rb
###
data:
......
......@@ -191,3 +191,18 @@ test_clang7_release:
extends:
- .testing-clang
- .testing_various_compilers_release
test_clang11_manual:
image: debian:bullseye-slim
variables:
CLANG_VERSION: "11"
extends:
- .testing-clang
- .testing_various_compilers_manual
test_clang11_release:
image: debian:bullseye-slim
variables:
CLANG_VERSION: "11"
extends:
- .testing-clang
- .testing_various_compilers_release
......@@ -277,7 +277,7 @@ RELEASE_ALGO = src/
RELEASE_TOOLS = $(wildcard ../tools/*.py)
RELEASE_MAKE = ../tools/tests/Makefile ../doc/Makefile
RELEASE_TESTS = ../doc/vidjil-format.md $(wildcard ../doc/analysis-example.vidjil) $(wildcard ../tools/tests/*.should_get) ../tools/should.py ../tools/diff_json.sh ../demo/get-sequences ../demo/Demo-X5.fa ../demo/Makefile
RELEASE_GERMLINES = ../germline/Makefile ../germline/germline_id ../germline/get-saved-germline ../germline/get-germline ../germline/split-from-imgt.py $(wildcard ../germline/*.g) ../germline/revcomp-fasta.py ../germline/fasta.py ../germline/homo-sapiens/IKZF*fa ../germline/homo-sapiens/ERG*fa
RELEASE_GERMLINES = ../germline/Makefile ../germline/germline_id ../germline/get-saved-germline ../germline/get-germline ../germline/split-germlines.py $(wildcard ../germline/*.g) ../germline/revcomp-fasta.py ../germline/fasta.py ../germline/homo-sapiens/IKZF*fa ../germline/homo-sapiens/ERG*fa
RELEASE_HELP = ../doc/vidjil-algo.md ../doc/locus.md ../doc/tools.md ../doc/vidjil-format.md ../doc/should-vdj.md ../doc/quality.md ../doc/credits.md ../doc/CHANGELOG ../doc/LICENSE ../README.md ../INSTALL.md
RELEASE_FILES = $(RELEASE_TOOLS) $(RELEASE_TESTS) $(RELEASE_MAKE) $(RELEASE_GERMLINES) $(RELEASE_HELP)
......
......@@ -29,6 +29,7 @@
#include <list>
#include <stdexcept>
#define STDIN_FILENAME "-"
#define SAMPLE_APPROX_NB_SEQUENCES 200
using namespace std;
......
......@@ -49,7 +49,11 @@ OnlineFasta::~OnlineFasta() {
}
void OnlineFasta::init() {
if (! filename.empty()) {
if (filename == STDIN_FILENAME) {
input = &cin;
input_allocated = false;
}
else if (! filename.empty()) {
input = new igzstream(filename.c_str());
if (this->input->fail()) {
......
#include "filter.h"
#include "math.hpp"
FilterWithACAutomaton::FilterWithACAutomaton(BioReader &origin, string seed) : originalBioReader(origin){
FilterWithACAutomaton::FilterWithACAutomaton(BioReader &origin, string seed, float keys_compress) : originalBioReader(origin){
this->filtered_sequences_nb = 0;
this->filtered_sequences_calls = 0;
buildACAutomatonToFilterBioReader(seed);
buildACAutomatonToFilterBioReader(seed, keys_compress);
}
FilterWithACAutomaton::~FilterWithACAutomaton(){
......@@ -16,9 +16,8 @@ FilterWithACAutomaton::~FilterWithACAutomaton(){
}
}
void FilterWithACAutomaton::buildACAutomatonToFilterBioReader(string seed){
char asciiChar;
int asciiNumber;
void FilterWithACAutomaton::buildACAutomatonToFilterBioReader(string seed, float keys_compress){
unsigned int asciiNumber;
string currentLabel;
string previousLabel;
......@@ -27,26 +26,29 @@ void FilterWithACAutomaton::buildACAutomatonToFilterBioReader(string seed){
indexes = nullptr;
return;
}
automaton = new PointerACAutomaton<KmerAffect>(seed, false, true);
automaton = new PointerACAutomaton<KmerStringAffect>(seed, false, true);
indexes = new vector<int>();
asciiNumber = SPECIFIC_KMERS_NUMBER;
automaton->insert(originalBioReader.sequence(0),std::string("") + char(asciiNumber), true, 0, seed);
automaton->insert(originalBioReader.sequence(0), std::to_string(asciiNumber), true, 0, seed);
indexes->push_back(0);
unsigned int previousAsciiNumber = asciiNumber;
int rawNumber = 0;
previousLabel = extractGeneName(originalBioReader.label(0));
for(int i = 1;i < originalBioReader.size(); ++i){
currentLabel = extractGeneName(originalBioReader.label(i));
if(currentLabel != previousLabel){
indexes->push_back(i);
asciiNumber++;
asciiNumber = SPECIFIC_KMERS_NUMBER + 1 + (int) rawNumber / keys_compress;
rawNumber++;
}
if(asciiNumber > 127){
delete automaton; delete indexes;
automaton = nullptr;
indexes = nullptr;
return;
if (asciiNumber > previousAsciiNumber)
{
indexes->push_back(i);
previousAsciiNumber = asciiNumber;
}
asciiChar = char(asciiNumber