Commit 8e1f0f4e authored by flothoni's avatar flothoni

Merge branch 'dev' of gitlab.inria.fr:vidjil/vidjil into feature-c/4020_minor_fix_distribution

parents 98d03ff5 10e44733
Pipeline #127474 failed with stages
in 10 minutes
......@@ -29,4 +29,6 @@ dep.mk
/browser/cgi/align.cgi
/browser/js/conf.js
node_modules/
cypress/
\ No newline at end of file
cypress/
server/web2py/applications/vidjil/errors
*.bak
......@@ -51,6 +51,11 @@ test_germlines:
- /^feature-.*g.*\/.*$/
- schedules
# Doc
include:
- local: '/doc/.gitlab-ci.yml'
- local: 'algo/.gitlab-ci-compilers.yml' # Stage multiple_tests
# Algorithm
......@@ -173,33 +178,6 @@ deploy_release_prod:
make -C vidjil-algo-${release_name};\
ln -sfT vidjil-algo-${release_name} next"
.testing-compilers: &test_various_compilers
stage: multiple_tests
tags:
- several-compilers
script:
- |
for CXX in $COMPILERS; do
echo "**** Using $CXX for compilation ****"
make -C algo clean
make -C algo CXX=$CXX unit
make -C algo CXX=$CXX should
make -C algo CXX=$CXX valgrind_unit
done
test_various_compilers_manual:
<<: *test_various_compilers
when: manual
only:
- /^feature-.*a.*\/.*$/
- schedules
test_various_compilers_release:
<<: *test_various_compilers
only:
- /^feature-.*a.*\/release$/
# Client
test_browser_unit:
......@@ -392,7 +370,7 @@ ff45-server-functional:
benchmark_algo:
stage: benchmark
script:
- cd algo/tests ; python3 benchmark-releases.py -bi
- cd algo/tests ; python3 benchmark-releases.py -bic
when: manual
only:
- /^feature-.*a.*\/.*$/
......
.testing-compilers:
stage: multiple_tests
tags:
- cidocker
before_script:
- apt-get update
- apt-get install -y time valgrind python3 wget tar
script:
- $CXX --version
- make demo data germline
- make -C algo clean
- make -C algo CXX=$CXX CC=$CC unit
- make -C algo CXX=$CXX CC=$CC should
- make -C algo CXX=$CXX CC=$CC valgrind_unit
.installing-compiler:
before_script:
- apt-get update
- apt-get install -y time valgrind python3 $COMPILER_PKG-$VERSION zlib1g-dev make wget tar python2.7
- ln -sf /usr/bin/python2.7 /usr/bin/python
- export CXX=${COMPILER_CPP}-$VERSION
- export CC=${COMPILER_C}-$VERSION
.testing-gcc:
extends: .testing-compilers
variables:
CXX: g++
CC: gcc
.testing-clang:
variables:
COMPILER_PKG: "clang"
COMPILER_CPP: "clang++"
COMPILER_C: "clang"
VERSION: $CLANG_VERSION
extends:
- .testing-compilers
- .installing-compiler # Overrides the before_script to install the compiler
.installing-testing-gcc:
variables:
COMPILER_PKG: "g++"
COMPILER_CPP: "g++"
COMPILER_C: "gcc"
VERSION: $GCC_VERSION
extends:
- .testing-compilers
- .installing-compiler
.testing_various_compilers_manual:
when: manual
only:
- /^feature-.*a.*\/.*$/
- schedules
.testing_various_compilers_release:
only:
- /^feature-.*a.*\/release$/
test_gcc48_manual:
image: debian:8-slim
variables:
GCC_VERSION: "4.8"
extends:
- .installing-testing-gcc
- .testing_various_compilers_manual
test_gcc48_release:
image: debian:8-slim
variables:
GCC_VERSION: "4.8"
extends:
- .installing-testing-gcc
- .testing_various_compilers_release
test_gcc5_manual:
image: gcc:5.3
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc5_release:
image: gcc:5.3
extends:
- .testing-gcc
- .testing_various_compilers_release
test_gcc6_manual:
image: gcc:6.3
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc6_release:
image: gcc:6.3
extends:
- .testing-gcc
- .testing_various_compilers_release
test_gcc7_manual:
image: gcc:7.3
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc7_release:
image: gcc:7.3
extends:
- .testing-gcc
- .testing_various_compilers_release
test_gcc8_manual:
image: gcc:8
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc8_release:
image: gcc:8
extends:
- .testing-gcc
- .testing_various_compilers_release
test_gcc9_manual:
image: gcc:9
extends:
- .testing-gcc
- .testing_various_compilers_manual
test_gcc9_release:
image: gcc:9
extends:
- .testing-gcc
- .testing_various_compilers_release
test_clang34_manual:
image: debian:8-slim
variables:
CLANG_VERSION: "3.4"
extends:
- .testing-clang
- .testing_various_compilers_manual
test_clang34_release:
image: debian:8-slim
variables:
CLANG_VERSION: "3.4"
extends:
- .testing-clang
- .testing_various_compilers_release
test_clang4_manual:
image: debian:8-slim
variables:
CLANG_VERSION: "4.0"
extends:
- .testing-clang
- .testing_various_compilers_manual
test_clang4_release:
image: debian:8-slim
variables:
CLANG_VERSION: "4.0"
extends:
- .testing-clang
- .testing_various_compilers_release
test_clang6_manual:
image: debian:8-slim
variables:
CLANG_VERSION: "6.0"
extends:
- .testing-clang
- .testing_various_compilers_manual
test_clang6_release:
image: debian:8-slim
variables:
CLANG_VERSION: "6.0"
extends:
- .testing-clang
- .testing_various_compilers_release
test_clang7_manual:
image: debian:10-slim
variables:
CLANG_VERSION: "7"
extends:
- .testing-clang
- .testing_various_compilers_manual
test_clang7_release:
image: debian:10-slim
variables:
CLANG_VERSION: "7"
extends:
- .testing-clang
- .testing_various_compilers_release
......@@ -278,7 +278,7 @@ RELEASE_TOOLS = $(wildcard ../tools/*.py)
RELEASE_MAKE = ../tools/tests/Makefile ../doc/Makefile
RELEASE_TESTS = ../doc/vidjil-format.md $(wildcard ../doc/analysis-example.vidjil) $(wildcard ../tools/tests/*.should_get) ../tools/should.py ../tools/diff_json.sh ../demo/get-sequences ../demo/Demo-X5.fa ../demo/Makefile
RELEASE_GERMLINES = ../germline/Makefile ../germline/germline_id ../germline/get-saved-germline ../germline/get-germline ../germline/split-from-imgt.py $(wildcard ../germline/*.g) ../germline/revcomp-fasta.py ../germline/fasta.py ../germline/homo-sapiens/IKZF*fa ../germline/homo-sapiens/ERG*fa
RELEASE_HELP = ../doc/vidjil-algo.md ../doc/locus.md ../doc/dev.org ../doc/should-vdj.org ../doc/credits.md ../doc/CHANGELOG ../doc/LICENSE ../README.md ../INSTALL.md
RELEASE_HELP = ../doc/vidjil-algo.md ../doc/locus.md ../doc/tools.md ../doc/vidjil-format.md ../doc/should-vdj.md ../doc/quality.md ../doc/credits.md ../doc/CHANGELOG ../doc/LICENSE ../README.md ../INSTALL.md
RELEASE_FILES = $(RELEASE_TOOLS) $(RELEASE_TESTS) $(RELEASE_MAKE) $(RELEASE_GERMLINES) $(RELEASE_HELP)
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
#include "output.h"
#define NULL_VAL ""
......@@ -129,12 +128,13 @@ CloneOutput* SampleOutput::getClone(junction junction)
// .vidjil json output
void SampleOutputVidjil::out(ostream &s)
void SampleOutputVidjil::out(ostream &s, bool with_clones)
{
json j_clones;
for (auto it: clones)
j_clones.push_back(it.second->toJson());
if (with_clones)
for (auto it: clones)
j_clones.push_back(it.second->toJson());
j["clones"] = j_clones;
......
......@@ -69,7 +69,7 @@ public:
class SampleOutputVidjil : public SampleOutput
{
public:
void out(ostream &s);
void out(ostream &s, bool with_clones=true);
};
// AIRR
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
/*
This file is part of Vidjil <http://www.vidjil.org>
Copyright (C) 2011-2019 by VidjilNet consortium and Bonsai bioinformatics
Copyright (C) 2011-2020 by VidjilNet consortium and Bonsai bioinformatics
at CRIStAL (UMR CNRS 9189, Université Lille) and Inria Lille
Contributors:
Mathieu Giraud <mathieu.giraud@vidjil.org>
......
......@@ -5,6 +5,8 @@ SRC = DEST + 'src/'
BIN = DEST + 'bin/'
RUN = DEST + 'run/'
CURRENT = 'HEAD'
#####
LIMIT1e5 = '-x 100000 '
......@@ -70,7 +72,8 @@ import resource
stats = {}
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--install', action='store_true', help='install various releases')
parser.add_argument('-c', '--current', action='store_true', help='install current HEAD')
parser.add_argument('-i', '--install', action='store_true', help='install various releases from %s' % ARCHIVE)
parser.add_argument('-b', '--benchmark', action='store_true', help='benchmark installed releases')
......@@ -127,6 +130,12 @@ def install(release, tgz):
go('mkdir -p %s' % dir)
log = dir + '/' + 'install.log'
if release == CURRENT:
go('make -C ../../algo', log)
go('cp ../../vidjil-algo %s/%s ' % (BIN, release), log)
return
go('wget %s/%s -O %s/src.tgz' % (ARCHIVE, tgz, dir), log)
go('cd %s ; tar xfz src.tgz' % dir, log)
go('cd %s/*%s* ; make vidjil-algo || make CXX=g++-6' % (dir, release), log)
......@@ -134,7 +143,10 @@ def install(release, tgz):
print()
def install_all():
def install_current():
install(CURRENT, None)
def install_from_archive():
for release, tgz in get_releases():
try:
install(release, tgz)
......@@ -197,8 +209,11 @@ if __name__ == '__main__':
if not args.install and not args.benchmark:
parser.print_help()
if args.current:
install_current()
if args.install:
install_all()
install_from_archive()
if args.benchmark:
bench_all()
......
# exact sequences
>TRDV1--DD3+d-exact
ttacagctagaagattcagcaaagtacttttgtgctc
CACAGTGCTACAAAACCTACAGAGACCTGTACAAA
# 3 mutations, no 9-mer
>TRDV1--DD3+d-no-9-mer
ttacagctagaagattcagcaaagtacttttgtgctc
CACAGTGCaACAAAACCcACAGAGACgTGTACAAA
# X X X
# 3 mutations, only one 9-mer
>TRDV1--DD3+d-one-9-mer
ttacagctagaagattcagcaaagtacttttgtgctc
CACAGTGCaACAAAACCTcCAGAGACCaGTACAAA
# X====9====X X
# random J sequence but with one spurious 9-mer
>TRDV1--random
ttacagctagaagattcagcaaagtacttttgtgctc
gctgatcataACAAAACCTctgtcagtcatgagctg
......@@ -6,7 +6,7 @@ $ All sequences have been seen as unique clone
$ All sequences have been seen as different IKZF1 recombinations
1: IKZF1 .* 22
### control sequence given by Debré ###
### control sequence given by Debre ###
# del4-7__jonction_intron3_intron7
$ Find the correct assignation for the sequence IKZF1-INTRON-3//IKZF1-INTRON-7
1: IKZF1-INTRON-3 16/CCCGCG/8 IKZF1-INTRON-7
......@@ -38,9 +38,9 @@ $ Find the correct assignation for the sequence
### 13922 - sequences ikfz1 ####
$ Find the correct assignation for the sequence (témoin del4-7)
$ Find the correct assignation for the sequence (temoin del4-7)
1: IKZF1-INTRON-3 1//2 IKZF1-INTRON-7
$ Find the correct assignation for the sequence (témoin del2-7)
$ Find the correct assignation for the sequence (temoin del2-7)
1: IKZF1-INTRON-1 8/CCCTTC/3 IKZF1-INTRON-7
$ Find the correct assignation for the sequence (seq 4-7 du patient)
1: IKZF1-INTRON-3 0/TCGG/6 IKZF1-INTRON-7
......
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -E 1.0 -g $VIDJIL_DIR/germline ../should-vdj-tests/Demo-X5.should-vdj.fa
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline ../should-vdj-tests/Demo-X5.should-vdj.fa
$ Detects a CDR3 on regular V(D)J recombinations
1: IGH SEG.* [{].*[}]
......
......@@ -3,7 +3,7 @@
# We launch three times a sequence of interest (buggy-D.fa) with a various
# number of distinct reads (mutations randomly inserted in the middle).
# The segmentation of one sequence should not depend on the number of the
# The designation of one sequence should not depend on the number of the
# other reads. This is what is tested, we first put 10 sequences, then 5 and
# finally just the sequence of interest alone.
......@@ -14,5 +14,5 @@
$ Three times the same window
3: TGTGCGGGATCTTCGTCCTCTTATCATAATAATGGTTTTTTGGCGGGGGAGTCATGGGGC
$ Three times the same segmentation
f3: IGHV4-34.*IGHD.*IGHJ4
$ Three times the same designation
3: IGHV4-34.*IGHD.*IGHJ4
### Testing -E (e-value threshold for D detection)
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations --first-reads 1 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRB ../should-vdj-tests/0000-nck-TRB.should-vdj.fa
$ Default -E value, no D here
1: TRBV6-1.* 7/AGGTGAGTCCC/2 TRBJ2-7
0: TRBD1
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations --first-reads 1 -e 1e6 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRB ../should-vdj-tests/0000-nck-TRB.should-vdj.fa
$ A large -e has no consequence on the default -E value, still no D here
1: TRBV6-1.* 7/AGGTGAGTCCC/2 TRBJ2-7
0: TRBD1
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations --first-reads 1 -E 100 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRB ../should-vdj-tests/0000-nck-TRB.should-vdj.fa
$ With -E 100, a D is detected
1: TRBV6-1.* TRBD1.* TRBJ2-7
###
!LAUNCH: $VIDJIL_DIR/$EXEC --all -g $VIDJIL_DIR/germline/homo-sapiens.g:TRD+ $VIDJIL_DATA/hard-kmers.fa
$ Detect 1 read
1:junction detected in 1 read
$ Detect the exact recombination
1:>clone.*exact
### -e-value 1e12
!LAUNCH: $VIDJIL_DIR/$EXEC --all --e-value 1e12 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRD+ $VIDJIL_DATA/hard-kmers.fa
$ Detect three sequences
1:junction detected in 3 reads
$ Detect the exact recombination
1:>clone.*exact
$ Detect the recombination with only one k-mer
1:>clone.*one-9-mer
$ Sadly, detect and designate the read with a random J but with a spurious k-mer
1:>clone.*random
1:>clone.*random.*TRDV1.* 10/.*/22 TRDD3
### --e-value-kmer 1e12
!LAUNCH: $VIDJIL_DIR/$EXEC --all --e-value-kmer 1e12 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRD+ $VIDJIL_DATA/hard-kmers.fa
$ Detect three sequences
1:junction detected in 3 reads
$ Detect the exact recombination
1:>clone.*exact
$ Detect the recombination with only one k-mer
1:>clone.*one-9-mer
$ Sadly, detect the read with a random J but with a spurious k-mer
1:>clone.*random
$ but... do not designate it, as the global --e-value is still the default 1.0
1:>clone.*random.*UNSEG
### --seed 12s
!LAUNCH: $VIDJIL_DIR/$EXEC -K --all --seed 12s -g $VIDJIL_DIR/germline/homo-sapiens.g:TRD+ $VIDJIL_DATA/hard-kmers.fa
$ With another seed, detect three sequences
1:junction detected in 3 reads
$ Do not detect the read with a random J
0:>clone.*random
......@@ -15,7 +15,7 @@ $ Do not display advanced options
0: custom Cost
$ Correct number of regular options
24:^..-
25:^..-
!NO_LAUNCHER:
......@@ -36,5 +36,5 @@ $ Display advanced options
: , experimental options
: custom Cost
$ Correct number of options
54:^..-
$ Correct number of options, including advanced options
57:^..-
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH,IGK,IGL $VIDJIL_DATA/multi-complete.fa ; cat out/multi-complete.vidjil | python $VIDJIL_DIR/tools/format_json.py -1
### Focusing on Ig recombinations with -g:IGH,IGK,IGL
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -e 0.1 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH,IGK,IGL $VIDJIL_DATA/multi-complete.fa ; cat out/multi-complete.vidjil | python $VIDJIL_DIR/tools/format_json.py -1
$ Segment the Ig recombinations
$ Detect the Ig recombinations
1:IGH SEG
1:IGK SEG
1:IGL SEG
$ Do not segment the TR recombinations
$ Do not detect the TR recombinations
4:TR.* UNSEG
$ Report the unsegmented sequences in the json output
$ Report the undetected sequences in the json output
1: "germline": "not analyzed", "id": "000001", "name": "TRA
1: "germline": "not analyzed", "id": "000002", "name": "TRB
1: "germline": "not analyzed", "id": "000003", "name": "TRG
1: "germline": "not analyzed", "id": "000004", "name": "TRD
$ Count the unsegmented sequences in the json output
$ Count the undetected sequences in the json output
1: "not analyzed": .4.
$ Count the segmented sequences in the json output
$ Count the detected sequences in the json output
1: "segmented": .3.
!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -e 0.1 -g $VIDJIL_DIR/germline $VIDJIL_DATA/multi-tiny.fa
$ Do not designate recombinations in these reads, they are too small
7:UNSEG
!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline $VIDJIL_DATA/multi-complete.fa
$ Segment all the seven reads
$ Detect recombinations in all the seven reads
1:junction detected in 7 reads
$ Segment one read on TRA
$ Detect one read on TRA
1:TRA .* -> .* 1