Merge branch 'master' of git+ssh://scm.gforge.inria.fr//gitroot/vidjil/vidjil

.travis.yml

@@ -36,5 +36,7 @@ after_success:
   - make should_coveralls
 
 notifications:
+  email:
+    - notifications@vidjil.org
   webhooks:
     - https://buildtimetrend.herokuapp.com/travis

algo/core/windowExtractor.cpp

 #include "windowExtractor.h"
 #include "segment.h"
 
+// Progress bar
+#define PROGRESS_POINT 25000
+#define PROGRESS_LINE 40
+
 WindowExtractor::WindowExtractor(): out_segmented(NULL), out_unsegmented(NULL), out_affects(NULL), max_reads_per_window(~0){}
                                     
 WindowsStorage *WindowExtractor::extract(OnlineFasta *reads, MultiGermline *multigermline,
@@ -18,6 +22,7 @@ WindowsStorage *WindowExtractor::extract(OnlineFasta *reads, MultiGermline *mult
     }
 
   int nb_reads_all = 0;
+  int bp_total = 0;
 
   while (reads->hasNext() && (int) nb_reads != stop_after) {
     reads->next();
@@ -74,7 +79,23 @@ WindowsStorage *WindowExtractor::extract(OnlineFasta *reads, MultiGermline *mult
     if (out_affects) {
       *out_affects << "#" << seg->getInfoLine() << endl;
     }
+
+    // Progress bar
+    bp_total += read_length;
+
+    if (!(nb_reads % PROGRESS_POINT))
+      {
+	cout << "." ;
+
+	if (!(nb_reads % (PROGRESS_POINT * PROGRESS_LINE)))
+	  cout << setw(10) << nb_reads / 1000 << "k reads " << fixed << setprecision(2) << setw(14) << bp_total / 1E6 << " Mbp" << endl ;
+
+	cout.flush() ;
+      }
   }
+
+  cout << endl ;
+
   return windowsStorage;
 }
 

browser/css/dark.css

@@ -180,6 +180,12 @@ line {
   opacity: 0.5 ;
   stroke-width: 20px;
 }
+.axis_m_other {
+  stroke: #839496;
+  opacity: 0.25 ;
+  stroke-width: 10px;
+  display: none;
+}
 .axis_h {
   stroke: #839496;
 }

browser/css/light.css

@@ -180,6 +180,12 @@ line {
   opacity: 0.5 ;
   stroke-width: 20px;
 }
+.axis_m_other {
+  stroke: #657b83;
+  opacity: 0.25 ;
+  stroke-width: 10px;
+  display: none;
+}
 .axis_h {
   stroke: #657b83;
 }

browser/css/vidjil.less

@@ -228,6 +228,13 @@ line {
   stroke-width : 20px;
 }
 
+.axis_m_other {
+  stroke: @default ;
+  opacity: 0.25 ;
+  stroke-width: 10px;
+  display: none;
+}
+
 .axis_h{
   stroke : @default ;
 }

browser/js/axis.js

@@ -117,7 +117,7 @@ Axis.prototype = {
     /* use clone size
      * 
      * */
-    useSize: function () {
+    useSize: function (other) {
         this.init()
         var self = this;
         this.sizeScale = d3.scale.log()
@@ -126,7 +126,8 @@ Axis.prototype = {
             
         //clone position
         this.pos = function(cloneID) {
-            var size = self.m.clone(cloneID).getSize()
+            var time = other ? self.m.tOther : self.m.t
+            var size = self.m.clone(cloneID).getSize(time)
             if (size !=0 ) {
                 return self.sizeScale(size);
             }else{

browser/js/graph.js

@@ -102,6 +102,7 @@ function Graph(id, model) {
     this.text_position_x = 60;
 
     this.m.view.push(this)
+    this.m.graph = this // TODO: find a better way to do this
 }
 
 Graph.prototype = {
@@ -1066,10 +1067,28 @@ Graph.prototype = {
             this.mobil.pos = this.graph_col[this.m.samples.order.indexOf(this.m.t)];
             this.data_axis.push(this.mobil)
         }
+
+        //other selected time_point
+        if ( this.m.samples.order.indexOf(this.m.tOther) != -1 ){
+            this.mobilOther = {};
+            this.mobilOther.type = "axis_m_other";
+            this.mobilOther.text = "";
+            this.mobilOther['class'] = "graph_text";
+            this.mobilOther.orientation = "vert";
+            this.mobilOther.pos = this.graph_col[this.m.samples.order.indexOf(this.m.tOther)];
+            this.data_axis.push(this.mobilOther)
+        }
         
         return this
     },
-    
+
+    setOtherVisibility: function (visible) {
+        if (visible) {
+            $(".axis_m_other").show("fast")
+        } else {
+            $(".axis_m_other").hide("fast")
+        }
+    },
 
 /* ************************************************ *
  * RENDERER FUNCTIONS

browser/js/model.js

@@ -72,7 +72,8 @@ Model.prototype = {
         this.data = {};
         this.data_info = {};
         
-        this.t = 0;
+        this.t = 0;          // Selected time/sample
+        this.tOther = 0;  // Other (previously) selected time/sample
         this.focus = -1;
         this.system_selected = []
         
@@ -389,6 +390,10 @@ Model.prototype = {
             self.samples.order = []
             for (var i = 0; i < self.samples.number; i++) self.samples.order.push(i);
         }
+        if (self.samples.order.length >= 2) {
+            self.tOther = 1
+        }
+
         if (typeof self.samples.names =='undefined'){
             self.samples.names = []
             for (var i = 0; i < self.samples.number; i++) self.samples.names.push("");
@@ -1479,6 +1484,7 @@ Model.prototype = {
      * */
     changeTime: function (newT) {
         console.log("changeTime()" + newT)
+        this.tOther = this.t
         this.t = newT;
         this.update();
         return this.t

browser/js/scatterPlot.js

@@ -103,7 +103,8 @@ function ScatterPlot(id, model) {
         ["gene_j", "gene J"],
         ["allele_v", "allele V"],
         ["allele_j", "allele J"],
-        ["Size", "abundance"],
+        ["Size", "size"],
+        ["otherSize", "size (other point)"],
         ["sequenceLength", "clone length"],
         ["GCContent", "GC content"],
         ["n", "N length"],
@@ -119,7 +120,8 @@ function ScatterPlot(id, model) {
         // "V/abundance" : { "mode": "plot", "x" : "gene_v", "y": "Size"},
         "V distribution" :            { "mode": "bar", "x" : "gene_v",         "y": "gene_j"},
         "Clone length distribution" : { "mode": "bar", "x" : "sequenceLength", "y": "gene_v"},
-        "N length distribution" :     { "mode": "bar", "x" : "n",              "y": "gene_v"}
+        "N length distribution" :     { "mode": "bar", "x" : "n",              "y": "gene_v"},
+        "compare two samples" : { "mode": "plot", "x" : "Size", "y": "otherSize"}
     };
     this.default_preset = 1
 
@@ -846,6 +848,9 @@ ScatterPlot.prototype = {
             case "gene_j" :
                 this.sortBarTab(function(cloneID){return self.m.clone(cloneID).getJ(false)});
                 break;
+            case "otherSize" :
+                this.sortBarTab(function(cloneID){return self.m.clone(cloneID).getSize(m.tOther)});
+                break;
             case "Size" :
                 this.sortBarTab(function(cloneID){return self.m.clone(cloneID).getSize()});
                 break;
@@ -1959,6 +1964,11 @@ ScatterPlot.prototype = {
         }else{
             this.update();
         }
+
+        if (typeof m.graph != "undefined") {
+            m.graph.setOtherVisibility(this.splitX == "otherSize" || this.splitY == "otherSize")
+        }
+
     },
 
     
@@ -1976,8 +1986,11 @@ ScatterPlot.prototype = {
             case "gene_j" :
                 axis.useGermline(this.m.germlineJ, "J", false)
                 break;
+            case "otherSize" :
+                axis.useSize(true)
+                break;
             case "Size" :
-                axis.useSize()
+                axis.useSize(false)
                 break;
             case "sequenceLength" :
                 axis.custom(function(cloneID) {

browser/js/segmenter.js

@@ -50,6 +50,8 @@ function Segment(id, model) {
     this.m.view.push(this); //synchronisation au Model
     this.starPath = "M 0,6.1176482 5.5244193, 5.5368104 8.0000008,0 10.172535,5.5368104 16,6.1176482 11.406183,9.9581144 12.947371,16 8.0000008,12.689863 3.0526285,16 4.4675491,10.033876 z"
     this.cgi_address = CGI_ADDRESS
+
+    this.first_clone = 0 ; // id of sequence at the top of the segmenter
     
     this.memtab = [];
     this.sequence = {};
@@ -447,6 +449,13 @@ Segment.prototype = {
         this.sequence[cloneID] = new Sequence(cloneID, this.m)
         
         var divParent = document.getElementById("listSeq");
+
+        // Am I the first clone in this segmenter ?
+        var previous_li = document.getElementById("listSeq").getElementsByTagName("li");
+        if (previous_li.length == 0) {
+            this.first_clone = cloneID
+        }
+
         var li = document.createElement('li');
         li.id = "seq" + cloneID;
         li.className = "sequence-line";
@@ -542,8 +551,8 @@ Segment.prototype = {
                 $(".seq-mobil").css({ opacity: 1 })
             },
             success: function (result) {
-                self.displayAjaxResult(result);
                 self.aligned = true ;
+                self.displayAjaxResult(result);
             },
             error: function () {
                 console.log({"type": "flash", "msg": "cgi error : impossible to connect", "priority": 2});
@@ -582,29 +591,30 @@ Segment.prototype = {
     resetAlign: function() {
         var selected = this.m.getSelected();
 
+        this.aligned = false
+
         for (var i = 0; i < selected.length; i++) {
             var spanM = document.getElementById("m" + selected[i])
             spanM.innerHTML =  this.sequence[selected[i]].load().toString()
         }
-
-        this.aligned = false
     },
     
     displayAjaxResult: function(file) {
 
         var json = JSON.parse(file)
 
+	// Load all (aligned) sequences
+        for (var i = 0; i < json.seq.length; i++) {
+            this.sequence[this.memTab[i]].load(json.seq[i])
+	}
+
+	// Render all (aligned) sequences
         for (var i = 0; i < json.seq.length; i++) {
 
             // global container
             var spanM = document.getElementById("m" + this.memTab[i]);
-            spanM.innerHTML = "";
-
             var seq = this.sequence[this.memTab[i]]
-            spanM.innerHTML = seq.load(json.seq[i])
-                .diff(json.seq[0])
-                .toString()
-
+            spanM.innerHTML = seq.toString()
         }
 
     },
@@ -805,13 +815,13 @@ Sequence.prototype = {
     },
 
     //compare sequence with another string and surround change
-    diff: function (str) {
-        for (var i = 0; i < this.seq.length; i++) {
-            if (this.seq[i] != str[i]) {
-                this.seq[i] = "<span class='substitution'>" + this.seq[i] + "</span>"
-            }
+    spanify_mutation: function (self, other) {
+        if (segment.aligned && self != other) {
+            return "<span class='substitution' other='" + other + '-' + segment.first_clone + "'>" + self + "</span>"
         }
-        return this;
+	else {
+            return self
+	}
     },
 
     //return sequence completed with html tag
@@ -876,9 +886,9 @@ Sequence.prototype = {
 
                 // one character
                 if (segment.amino) {
-                    result += this.seqAA[i]
+                    result += this.spanify_mutation(this.seqAA[i], segment.sequence[segment.first_clone].seqAA[i])
                 }else{
-                    result += this.seq[i]
+                    result += this.spanify_mutation(this.seq[i], segment.sequence[segment.first_clone].seq[i])
                 }
 
                 // VDJ spans - end

doc/browser-dev.org

-#+TITLE: Vidjil Algorithm -- Browser development documentation
+#+TITLE: Vidjil Browser -- development documentation
 #+AUTHOR: The Vidjil team (Marc, Mathieu, Mikaël and Tatiana)
 
 
@@ -21,7 +21,7 @@ as well with the locus selection.
 - The link with the patient database/server is done with the =Database= object (=js/database.js=)
 
 - Other objects: =Report=, =Shortcut=
-  Extends functionalities but cannot be currently used outside the default vidjil browser.
+  Extends functionalities but requires elements from the full =index.html=.
 
 
 ** Integrating the browser
@@ -37,10 +37,30 @@ as well with the locus selection.
   - The wonderful library =require.js= is used, so there is only one file to include
     <script data-main="js/app.js" src="js/lib/require.js"></script>
 
-  - =js/main.js= is the main file that creates the different views and binds them to the model.
-    Or you can define a function named "main()" and use it to build your own interface (see =small_example.html=)
+  - =js/main.js= creates the different views and binds them to the model.
+    Another option is to directly define a function named =main()=, as in =small_example.html=.
 
+*** JSON .vidjil data
 
+Clone lists can be passed to the model through several ways:
+  - directly by the user (import/export)
+  - from a patient database (needs a database)
+  - trough the API (see below)
+  - or by directly providing data through Javascript (as in =small_example.html=)
 
+The first three solutions needs some further elements from the full =index.html=.
 
 
+**** Browser API
+
+The browser can be opened on a data file specified from a =data= attribute,
+and optionally on an analysis file specified from a =analysis= attribute,
+as in the following URLs on our test server:
+
+- http://rbx.vidjil.org/browser/?data=test.vidjil
+- http://rbx.vidjil.org/browser/?data=test.vidjil&analysis=test.analysis
+- http://rbx.vidjil.org/browser/?data=http://rbx.vidjil.org/browser/test.vidjil
+
+Both GET and POST requests are accepted.
+Note that the =browser/index.html= file and the =.vidjil/.analysis= files should be hosted on the same server.
+Otherwise, the server hosting the =.vidjil/.analysis= files must accept cross-domain queries.

doc/browser.org

@@ -319,20 +319,6 @@ There can be several causes leading to bad ratios:
 
 
 
-* Browser API
-
-The browser can be opened on a data file specified from a =data= attribute, 
-and optionally on an analysis file specified from a =analysis= attribute,
-as in the following URLs on our test server:
-
-- http://rbx.vidjil.org/browser/?data=test.vidjil
-- http://rbx.vidjil.org/browser/?data=test.vidjil&analysis=test.analysis
-- http://rbx.vidjil.org/browser/?data=http://rbx.vidjil.org/browser/test.vidjil
-
-Both GET and POST requests are accepted.
-Note that the =browser/index.html= file and the =.vidjil/.analysis= files should be hosted on the same server.
-Otherwise, the server hosting the =.vidjil/.analysis= files must accept cross-domain queries.
-
 
 * Reference
 

germline/fasta.py

+
+
+
+def parse(fasta, endline=''):
+    '''Iterates over sequences in a fasta files, yielding (header, sequence) pairs'''
+
+    header = ''
+    sequence = ''
+    
+    for l in fasta:
+        l = l.strip()
+
+        if not l:
+            continue
+    
+        if l[0] == '>':
+            if header or sequence:
+                yield (header, sequence)
+            header = l[1:]
+            sequence = ''
+
+        else:
+            sequence += l + endline
+            
+    if header or sequence:
+        yield (header, sequence)
+        
+def parse_as_Fasta(fasta):
+    for (header, sequence) in parse(fasta):
+        yield Fasta(header, sequence)
+
+
+class Fasta():
+
+    def __init__(self, header, sequence):
+        self.header = header
+        self.seq = sequence
+
+    @property
+    def name(self):
+        return self.header.split('|')[1]
+
+    @property
+    def species(self):
+        return self.header.split('|')[2]
+
+    def __len__(self):
+        return len(self.seq)
+    
+    def __str__(self):
+        return '>%s\n%s\n' % (self.header, self.seq)
+    

germline/split-from-imgt.py

@@ -47,6 +47,8 @@ SPLIT_SEQUENCES = {'/DV': ['TRAV', 'TRDV']}
 SPECIES = {
     "Homo sapiens": './', 
     "Mus musculus": 'mus-musculus/',
+    "Rattus norvegicus": 'rattus-norvegicus/',
+    "Rattus norvegicus_BN/SsNHsdMCW": 'rattus-norvegicus/',
 }
 
 for l in sys.stdin:

server/web2py/applications/vidjil/modules/vidjil_utils.py

@@ -127,7 +127,7 @@ def log_links(s):
         start += 1
         end -= 1
 
-    elif task:
+    if task:
         call = "admin/log"
         args = {'file': '../../' + defs.DIR_OUT_VIDJIL_ID % task + defs.BASENAME_OUT_VIDJIL_ID % task + '.vidjil.log', 'format': 'raw'}
         (start, end) = m_task.span()