Merge branch 'dev' into 'feature-c/4014-clones-selectionnable-even-if-disable'

# Conflicts:
#   browser/test/functional/test_scatterplot.rb

.gitlab-ci.yml

@@ -127,6 +127,7 @@ algo_valgrind_functional:
 
 release_valgrind_functional:
   <<: *valgrind_functional
+  allow_failure: true
   only:
     - /^feature-.*a.*\/release$/
 
@@ -301,8 +302,8 @@ code_quality:
 test_server_unit:
     stage: test_unit
     script:
-      - virtualenv $CI_BUILD_REF_SLUG
-      - source $CI_BUILD_REF_SLUG/bin/activate
+      - virtualenv $CI_COMMIT_SHORT_SHA
+      - source $CI_COMMIT_SHORT_SHA/bin/activate
       - pip install -r requirements.txt
       - make -C server install_web2py_standalone
       - make -C server launch_fuse_server
@@ -457,12 +458,7 @@ deploy_prod:
   stage: deploy_prod
   script:
     - ssh $PROD_CLIENT_SERVER "
-         cd $PROD_CLIENT_PATH;
-         git fetch
-         && git reset --hard origin/prod-client
-         && make -C browser
-         && make -C browser sha1
-         && cp doc/analysis-example.vidjil browser/"
+         cd $PROD_CLIENT_PATH;"
   environment:
     name: production
     url: http://$PROD_CLIENT_SERVER/?data=analysis-example.vidjil

.gitlab/merge_request_templates/Release (vidjil-algo).md

 
-## Preparation
+## Before the release
+
+### Documentation
 
 * [ ] New features are described in `doc/vidjil-algo.md`
-* [ ] Breaking changes (or needed) are understood
+* [ ] Breaking changes (and needed configuration changes) are understood
  
 On the `feature-a/release` branch, the last commit is the release commit updating the three following files:
  * [ ] CHANGELOG
  * [ ] `doc/vidjil-algo.md` with the proper release tag
  * [ ] `algo/release` with the proper release tag
 
-## Pipelines
+### Pipelines
 
 https://gitlab.inria.fr/vidjil/vidjil/pipelines/XXXXX
+(if tests passed on different pipelines, indicate below the relevant pipelines)
 
 Usual tests should pass, but also:
 * [ ] prepare_release
@@ -23,21 +26,20 @@ Benchmarks
 * [ ] almost no change...
 * [ ] ... or significant changes are understood
 
-## Tag and push
+## The release, publish, tag and push
+
+* [ ] merge this MR to *master* and tag: `git tag release-20XX-XX` 
+* [ ] push: `git push origin master release-20XX-XX`
+* [ ] mirror to GH: `git push github master release-20XX.XX`
+* [ ] click `publish_release` (there may be changes to CD...)
 
-* [ ] tag (`git tag release-20XX-XX; git push origin release-20XX-XX`)
-* [ ] mirror to GH
+After these steps, we merge back the release in `dev`:
+* [ ] merge this MR to dev
 
-## Deploy
+## After the release: deploy
 
-* [ ] doc 
-* [ ] `app` (vidjil-algo-next)
-* [ ] `app` if needed, update server configs
-* [ ] `app` notification to users
-* [ ] `app` (vidjil-algo)
-* [ ] `app` test on production (X5 and L4)
-* [ ] `hds` queue for qualification ?
-* [ ] twice a year, communicate to users
+Sometimes, the deploy stages can occur several days after the release.
+Open a new internal issue with the "Deploy" template.
 
 /label ~cpp
 

algo/release

-2020.06
+2020.08

algo/tests/bugs/bug20141024.should

 !NO_LAUNCHER:
-!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full  $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
 !REQUIRES: which valgrind
+!LAUNCH: valgrind --suppressions=gzstream_cpp.supp --suppressions=libstdc++_leak.supp --leak-check=full  $VIDJIL_DIR/$EXEC --clean-memory -c clones -d -g ../../../germline/homo-sapiens.g:IGH -z 2 -r 1 bug20141024.fa 2>&1
+
 
 $ No invalid read with short sequences
 e1:ERROR SUMMARY: 0 errors from 0 contexts

algo/tests/bugs/bug4225.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
 !OUTPUT_FILE: out/bug4225-j.affects
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -1 -2 -K bug4225-j.fa
 
 $ Find only +k and ? affects before the stretch of _ for all loci
 16: seed .*(\+k| \?){28}( _)+$
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
 !OUTPUT_FILE: out/chimera-fake-half.affects
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline -r 1 -4 -K ../data/chimera-fake-half.fa
 
 $ Find only +B and ? affects on the TRB and unexpected lines
 2: seed .* _(\+B| \?){48} _

algo/tests/should-get-tests/algo-then-fuse.should

 !REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
-!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/fused.data ; cat out/fused.data | python $VIDJIL_DIR/tools/format_json.py -1
+
+!OUTPUT_FILE: out/algo-then-fuse.vidjil
+!OPTIONS: --mod jR
+
+!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -z 0 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/fuse.py out/Stanford_S22.vidjil out/Stanford_S22.vidjil -o out/algo-then-fuse.vidjil
 
 $ Points list
-1:"original_names": \[".*data//Stanford_S22.fasta", ".*data//Stanford_S22.fasta"\]
+2:samples.original_names
+:samples.original_names[0]: data//Stanford_S22.fasta
+:samples.original_names[1]: data//Stanford_S22.fasta
 
-$ Most abundant window, twice, fused 
-1:"id": "CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC".*"reads": \[8, 8\].*"top": 2
+$ Most abundant window, twice, fused
+:clones[0].id: CCACCTATTACTGTACCCGGGAGGAACAATATAGCAGCTGGTACTTTGACTTCTGGGGCC
+:clones[0].reads: [8, 8]
+:clones[0].top: 1
 
 $ Windows that are not in the bottom of the distribution
-1: "distribution": \{[^}]+ "0.00001": \[9062, 9062\]
+#  :reads.distribution.0.00001:  should#59
+:reads.distribution: [9062, 9062]
\ No newline at end of file

algo/tests/should-get-tests/algo-then-vidjil-to-fasta.should

 !REQUIRES: python $VIDJIL_DIR/tools/check_python_version.py
-!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py  -o out/S22.fasta out/Stanford_S22.vidjil ;
 !OUTPUT_FILE: out/S22.fasta
+!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -z 5 -w 60 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; python $VIDJIL_DIR/tools/vidjil-to-fasta.py  -o out/S22.fasta out/Stanford_S22.vidjil ;
+
 
 $ 5 representative sequences in the FASTA output file
 5:>

algo/tests/should-get-tests/analysis-cost.should

-!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC --analysis-cost '1, 2, 3, 4, 5' $VIDJIL_DATA/Stanford_S22.fasta
+
 
 $Check that correct custom cost is used
 1:use custom Cost "1, 2, 3, 4, 5"

algo/tests/should-get-tests/cdr3-stopcodon.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
 !OUTPUT_FILE: out/cdr3-stopcodon.vidjil
+!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/cdr3-stopcodon.fa
 
 $ Two identical junctions in JSON
 2: "CATWDRKNYYKKLF"

algo/tests/should-get-tests/config-write-then-read.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y 5 -z 1 --out-vdjfa -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
 
 # Testing -x/-y/-z options
 

algo/tests/should-get-tests/consensus-random-read-chooser.should

 !LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG --consensus-on-longest-sequences $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-longest.log
 !LAUNCH: $VIDJIL_DIR/$EXEC -w 20 -g $VIDJIL_DIR/germline/homo-sapiens.g:TRG $VIDJIL_DATA/test-random-consensus.fa.gz > consensus-random.log
 !NO_LAUNCHER:
-!LAUNCH: diff consensus-longest.log consensus-random.log
 !EXIT_CODE: 1
+!LAUNCH: diff consensus-longest.log consensus-random.log
 
 $ Output should differ: ReadQualityScore gives a consensus of 52bp (with the spurious insertion)
 # Appears twice in the header of the consensus sequence and in the similarity matrix

algo/tests/should-get-tests/error-deprecated-options.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -c segment -aAtl reads 2>&1
+
 
 $ Deprecated options
 5:is deprecated

algo/tests/should-get-tests/error-unknown-option.should

-!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
 !EXIT_CODE: 109
+!LAUNCH: $VIDJIL_DIR/$EXEC --hello reads 2>&1
+
 
 $ Unknown option
 1:error.* --hello

algo/tests/should-get-tests/germlines-error-no-germline.should

 !NO_LAUNCHER:
-!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 
 $ Error, no germlines
 1:error.* one germline must be given

algo/tests/should-get-tests/germlines-error.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/Makefile $VIDJIL_DATA/Stanford_S22.fasta 2>&1
+
 
 $ Error, incorrect *.g
 1:error.* cannot properly read

algo/tests/should-get-tests/germlines-several.should

@@ -13,9 +13,8 @@ $ Segment two reads on IgVC/IgJC (second -g option)
 1: IgVC .* -> .* 1
 1: IgJC .* -> .* 1 
 
-
-!LAUNCH: cat out/isotypes.vidjil
 !NO_LAUNCHER:
+!LAUNCH: cat out/isotypes.vidjil
 
 $ Both germlines are in .vidjil
 1: /germline"

algo/tests/should-get-tests/gzip-out.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a $VIDJIL_DATA/clones_simul.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH -b out-a --out-vdjfa $VIDJIL_DATA/clones_simul.fa
 
 $ Output
 1: out-a.vidjil
 1: out-a.tsv
 1: out-a.vdj.fa
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b $VIDJIL_DATA/clones_simul.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --gz -b out-b --out-vdjfa $VIDJIL_DATA/clones_simul.fa
 
 $ Compressed output
 1: out-b.vidjil.gz

algo/tests/should-get-tests/help-examples.should

 !NO_LAUNCHER:
 
-cd $VIDJIL_DIR
-./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' | sh
+cd $VIDJIL_DIR ; ./$EXEC -h | grep '$EXEC -c' | sed 's/X 50/X 5/' | sed 's/demo.LIL-L4/-X 1000 demo\/LIL-L4/' > help-examples.sh
+cd $VIDJIL_DIR ; sh help-examples.sh
 
 # Test examples embedded in './vidjil-algo -h'
 

algo/tests/should-get-tests/help.should

@@ -37,4 +37,4 @@ $ Display advanced options
 : custom Cost
 
 $ Correct number of options, including advanced options
-58:^..-
+60:^..-

algo/tests/should-get-tests/labels-json.should

@@ -9,8 +9,8 @@ $ Report two clones, even with --max-consensus 0
 : ==> 2 clones
 
 
-!LAUNCH: cat out/Stanford_S22.vidjil
 !NO_LAUNCHER:
+!LAUNCH: cat out/Stanford_S22.vidjil
 
 $ Labels are in the .json output 
 1: "label": "lab1"

algo/tests/should-get-tests/no-output.should

@@ -14,6 +14,12 @@ $ There are no clones in the .vidjil
 $ Correct output message
 1:only metadata
 
+$ There is no clone output in individual files
+0:detail, by clone
+
+$ There is no deprecated .vdj.fa file
+0:.vdj.fa
+
 !LAUNCH: rm out/Stanford_S22.tsv ; $LAUNCHER $VIDJIL_DIR/$EXEC $EXTRA -z 0 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta > /dev/null ; touch out/Stanford_S22.tsv ; cat out/Stanford_S22.tsv
 
 $ The AIRR .tsv file has four lines

algo/tests/should-get-tests/productive_stop_after_J.should

@@ -2,8 +2,8 @@
 ## Now a productive sequence, with a stop codon after a TRGJ gene
 ## We now use -J ../TRGJ.fa, see #3147
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -V $VIDJIL_DIR/germline/homo-sapiens/TRGV.fa -J $VIDJIL_DIR/germline/homo-sapiens/TRGJ.fa $VIDJIL_DATA/productive_stop_after_J.fa
 !OUTPUT_FILE: out/productive_stop_after_J.vidjil
+!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -V $VIDJIL_DIR/germline/homo-sapiens/TRGV.fa -J $VIDJIL_DIR/germline/homo-sapiens/TRGJ.fa $VIDJIL_DATA/productive_stop_after_J.fa
 
 $ Clone name
 1: "TRGV9.01 0/CA/0 TRGJP.01"

algo/tests/should-get-tests/productive_stop_before_V.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/productive_stop_before_V.fa
 !OUTPUT_FILE: out/productive_stop_before_V.vidjil
+!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -3 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/productive_stop_before_V.fa
 
 $ first clone name
 1: "IGHV1-69.06 2/AACCCCCAACAAAGC/4 IGHD3-3.01 9/CCCATAAGTACCGT/1 IGHJ6.02"

algo/tests/should-get-tests/revcomp-S22.should

 !LAUNCH: $LAUNCHER $VIDJIL_DIR/$EXEC -z 1 -k 9 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --ratio 0.001 -r 2 -x 1000 -y 1 -c clones $VIDJIL_DATA/Stanford_S22.fasta | sed 's/--IGH--.*VDJ\\(.*\\).$/\\1/' | sed 's/IGH SEG_./IGH SEG_X/' > vidjil_s22.log
 !LAUNCH: $LAUNCHER $VIDJIL_DIR/$EXEC -z 1 -k 9 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH --ratio 0.001 -r 2 -x 1000 -y 1 -c clones $VIDJIL_DATA/Stanford_S22.rc.fasta | sed 's/--IGH--.*VDJ\\(.*\\).$/\\1/' |  sed 's/IGH SEG_./IGH SEG_X/' > vidjil_s22_rc.log
-!LAUNCH: diff out/Stanford_S22{,.rc}.vidjil | grep GGG && diff vidjil_s22.log vidjil_s22_rc.log
+
 !EXIT_CODE: 1
+!LAUNCH: diff out/Stanford_S22{,.rc}.vidjil | grep GGG && diff vidjil_s22.log vidjil_s22_rc.log
 
 $ Same number detected
 0:==> detected

algo/tests/should-get-tests/trb-only-VJ.should

@@ -18,8 +18,8 @@ $ There is a warning on multiple candidate assignations
 $ The warning is also found on stdout
 1:\[warning\] \(W69\) Several genes with equal probability
 
-!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -g $VIDJIL_DIR/germline/homo-sapiens.g:TRB $VIDJIL_DATA/trb-only-VJ.fa
 !OUTPUT_FILE: out/trb-only-VJ.vidjil
+!LAUNCH: $VIDJIL_DIR/$EXEC -c designations -g $VIDJIL_DIR/germline/homo-sapiens.g:TRB $VIDJIL_DATA/trb-only-VJ.fa
 
 $ With -c designation, the recombination is properly designated, without any D
 1:"name": "TRBJ2-3.01"

algo/tests/should-get-tests/w-small-error.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -x 1 -w -10 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -x 1 -w -10 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 
 $ Error, too small -w
 1:error.* Too small -w

algo/tests/should-get-tests/w-small.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -y 1 -z 0 -w 10 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 !OUTPUT_FILE: out/seq/clone.fa-1
-### !EXIT_CODE: 1
+!LAUNCH: $VIDJIL_DIR/$EXEC -x 100 -r 1 -y 1 -z 0 -w 10 --out-clone-files -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta 2>&1
 
 $ We should find a window
 1:[ACGT]{10}

algo/tests/should-get-tests/xyz-y-all.should

-!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y all -z 1 -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
+!LAUNCH: $VIDJIL_DIR/$EXEC -r 1 -x 10 -y all -z 1 --out-vdjfa -g $VIDJIL_DIR/germline/homo-sapiens.g:IGH $VIDJIL_DATA/Stanford_S22.fasta ; cat out/Stanford_S22.vdj.fa
 
 # Testing -x/-y/-z options
 

algo/vidjil.cpp

@@ -97,6 +97,7 @@ enum { CMD_WINDOWS, CMD_CLONES, CMD_SEGMENT, CMD_GERMLINES } ;
 
 // Fixed filenames/suffixes
 #define CLONES_FILENAME ".vdj.fa"
+#define CLONE_DIR "seq/"
 #define CLONE_FILENAME "clone.fa-"
 #define WINDOWS_FILENAME ".windows.fa"
 #define SEGMENTED_FILENAME ".detected.vdj.fa"
@@ -566,7 +567,7 @@ int main (int argc, char **argv)
     -> group(group);
 
   bool output_sequences_by_cluster = false;
-  app.add_flag("--out-reads", output_sequences_by_cluster, "output all reads by clones (" CLONE_FILENAME "*), to be used only on small datasets") -> group(group);
+  app.add_flag("--out-reads", output_sequences_by_cluster, "output all reads by clones (in " CLONE_DIR "/" CLONE_FILENAME "* files), to be used only on small datasets") -> group(group);
 
   bool output_affects = false;
   app.add_flag("--out-affects,-K", output_affects,
@@ -586,6 +587,16 @@ int main (int argc, char **argv)
   bool out_gz = false;
   app.add_flag("--gz", out_gz, "output compressed .tsv.gz, .vdj.fa.gz, and .vidjil.gz files") -> group(group) -> level();
 
+  bool output_vdjfa = false;
+  app.add_flag("--out-vdjfa", output_vdjfa,
+               "output clones in a " CLONES_FILENAME " file (only for clone sequence data)")
+    -> group(group) -> level();
+
+  bool output_clone_files = false;
+  app.add_flag("--out-clone-files", output_clone_files,
+               "output clones in individual files (in " CLONE_DIR "/" CLONE_FILENAME "* files)")
+    -> group(group) -> level();
+
   bool no_airr = false;
   bool no_vidjil = false;
   app.add_flag("--no-airr", no_airr, "do not output AIRR .tsv") -> group(group) -> level();
@@ -700,7 +711,7 @@ int main (int argc, char **argv)
   for(string lab : windows_labels_explicit)
     windows_labels[lab] = string("--label");
   
-  string out_seqdir = out_dir + "/seq/" ;
+  string out_seqdir = out_dir + "/" + CLONE_DIR ;
 
   if (verbose)
     cout << "# verbose " << verbose << endl ;
@@ -731,10 +742,16 @@ int main (int argc, char **argv)
     return 2;
   }
 
-  const char *outseq_cstr = out_seqdir.c_str();
-  if (mkpath(outseq_cstr, 0755) == -1) {
-    cerr << ERROR_STRING << "Directory creation: " << out_seqdir << endl; perror("");
-    return 2;
+  if (output_sequences_by_cluster)
+    output_clone_files = true;
+
+  if (output_clone_files)
+  {
+    const char *outseq_cstr = out_seqdir.c_str();
+    if (mkpath(outseq_cstr, 0755) == -1) {
+      cerr << ERROR_STRING << "Directory creation: " << out_seqdir << endl; perror("");
+      return 2;
+    }
   }
 
   // Compute basename if not given as an option
@@ -1302,11 +1319,19 @@ int main (int argc, char **argv)
     cout << "  ==> suggested edges in " << out_dir+ f_basename + EDGES_FILENAME
         << endl ;
 
-    cout << "  ==> " << f_clones << "   \t(for post-processing with other software)" << endl ;
-    ostream* out_clones = new_ofgzstream(f_clones.c_str(), out_gz) ;
+    ostream* out_clones = NULL;
+    if (output_vdjfa)
+    {
+      cout << "  ==> " << f_clones << "   \t(for sequence post-processing with other software)" << endl;
+      cout << "!! To get structured data, do not parse the Fasta headers, but rather work on the .vidjil file." << endl;
+      out_clones = new_ofgzstream(f_clones.c_str(), out_gz);
+    }
 
-    cout << "  ==> " << out_seqdir + CLONE_FILENAME + "*" << "\t(detail, by clone)" << endl ; 
-    cout << endl ;
+    if (output_clone_files)
+    {
+      cout << "  ==> " << out_seqdir + CLONE_FILENAME + "*" << "\t(detail, by clone)" << endl ;
+      cout << endl ;
+    }
 
     global_interrupted = false;
     signal(SIGINT, sigintHandler);
@@ -1367,7 +1392,8 @@ int main (int argc, char **argv)
         // If max_representatives is reached, we stop here but still outputs the window
         if ((max_representatives >= 0) && (num_clone >= max_representatives + 1))
           {
-            *out_clones << window_str << endl ;
+            if (output_vdjfa)
+              *out_clones << window_str << endl ;
             continue;
           }
       }
@@ -1391,15 +1417,21 @@ int main (int argc, char **argv)
 
 
       //$$ Open CLONE_FILENAME
+      ofstream *out_clone = NULL;
 
-      string clone_file_name = out_seqdir+ CLONE_FILENAME + string_of_int(num_clone) ;
-      ofstream out_clone(clone_file_name.c_str());
+      if (output_clone_files)
+      {
+        string clone_file_name = out_seqdir+ CLONE_FILENAME + string_of_int(num_clone) ;
+        out_clone = new ofstream(clone_file_name.c_str());
 
+        *out_clone << window_str ;
+      }
 
       //$$ Output window
       if (clone_on_stdout)
+      {
         cout << window_str ;
-      out_clone << window_str ;
+      }
 
 	//$$ Compute a representative sequence
 	// Display statistics on auditionned sequences
@@ -1464,7 +1496,16 @@ int main (int argc, char **argv)
           {
             if (clone_on_stdout)
               cout << representative << endl ;
-            *out_clones << representative << endl ;
+
+            if (output_vdjfa)
+              *out_clones << representative << endl ;
+
+            if (output_clone_files)
+            {
+              out_clone->close();
+              delete out_clone;
+            }
+
             continue;
           }
 
@@ -1490,8 +1531,12 @@ int main (int argc, char **argv)
 	// to stdout, CLONES_FILENAME, and CLONE_FILENAME-*
   if (clone_on_stdout)
     cout << seg << endl ;
-	out_clone << seg << endl ;
-	*out_clones << seg << endl ;
+
+  if (output_clone_files)
+    *out_clone << seg << endl ;
+
+  if (output_vdjfa)
+    *out_clones << seg << endl ;
     
         seg.toOutput(clone);
 
@@ -1519,13 +1564,16 @@ int main (int argc, char **argv)
                 }
 
 	      // Output best V, (D) and J germlines to CLONE_FILENAME-*
+        if (output_clone_files)
+        {
               if ((segmented_germline->seg_method == SEG_METHOD_53) || (segmented_germline->seg_method == SEG_METHOD_543))
-                out_clone << ">" << seg.box_V->ref_label << endl << seg.box_V->ref << endl ;
+                *out_clone << ">" << seg.box_V->ref_label << endl << seg.box_V->ref << endl ;
               if ((segmented_germline->seg_method == SEG_METHOD_543) || (segmented_germline->seg_method == SEG_METHOD_ONE))
-                out_clone << ">" << seg.box_D->ref_label << endl << seg.box_D->ref << endl ;
+                *out_clone << ">" << seg.box_D->ref_label << endl << seg.box_D->ref << endl ;
               if ((segmented_germline->seg_method == SEG_METHOD_53) || (segmented_germline->seg_method == SEG_METHOD_543))
-                out_clone << ">" << seg.box_J->ref_label << endl << seg.box_J->ref << endl ;
-	      out_clone << endl;
+                *out_clone << ">" << seg.box_J->ref_label << endl << seg.box_J->ref << endl ;
+              *out_clone << endl;
+        }
 	   } // end if (seg.isSegmented())
 
         seg.checkWarnings(clone, clone_on_stdout);
@@ -1536,18 +1584,26 @@ int main (int argc, char **argv)
 	    
 	    for (list<Sequence>::const_iterator itt = sequences.begin(); itt != sequences.end(); ++itt)
 	      {
-		out_clone << *itt ;
+          *out_clone << *itt ;
 	      }
 	  }
 	
   if (clone_on_stdout)
     cout << endl ;
-      out_clone.close();
+
+      if (output_clone_files)
+      {
+        out_clone->close();
+        delete out_clone;
+      }
+
     } // end for clones
     signal(SIGINT, SIG_DFL);
 
     out_edges.close() ;
-    delete out_clones;
+
+    if (output_vdjfa)
+      delete out_clones;
 
     if (num_clone > last_num_clone_on_stdout)
       {

browser/css/dark.css

@@ -1155,9 +1155,11 @@ span.logo {
   text-align: center;
   vertical-align: middle;
 }
-#info_timepoint td {
+#info_timepoint,
+#info_timepoint td,
+.info_overlaps td {
   min-width: 150px;
-  max-width: 700px;
+  max-width: 100%;
   overflow-wrap: break-word;
   word-wrap: break-word;
   border-style: solid;
@@ -1166,16 +1168,22 @@ span.logo {
   vertical-align: middle;
 }
 #info_window table,
-#info_timepoint table {
+#info_timepoint table,
+.info_overlaps {
   border-style: solid;
   border-color: #333333;
   border-collapse: collapse;
 }
 #info_window table .header,
-#info_timepoint table .header {
+#info_timepoint table .header,
+.info_overlaps table .header {
   border-width: 2px;
   background-color: #333333;
 }
+.info_overlaps_line {
+  background-color: #333333;
+  font-weight: bold;
+}
 span.ok {
   color: green;
 }
@@ -1820,6 +1828,10 @@ table.log textarea {
 .tagColor8 {
   background: #839496;
 }
+.tagColor9 {
+  background: #bdbdbd;
+  margin-left: 7px;
+}
 .inactiveTag {
   background-image: linear-gradient(45deg, transparent -25%, #000000 100%, #000000 100%, transparent 0%);
 }

browser/css/light.css

@@ -1156,9 +1156,11 @@ span.logo {
   text-align: center;
   vertical-align: middle;
 }
-#info_timepoint td {
+#info_timepoint,
+#info_timepoint td,
+.info_overlaps td {
   min-width: 150px;
-  max-width: 700px;
+  max-width: 100%;
   overflow-wrap: break-word;
   word-wrap: break-word;
   border-style: solid;
@@ -1167,16 +1169,22 @@ span.logo {
   vertical-align: middle;
 }
 #info_window table,
-#info_timepoint table {
+#info_timepoint table,
+.info_overlaps {
   border-style: solid;
   border-color: #cccccc;
   border-collapse: collapse;