Commit bc364a73 authored by Mathieu Giraud's avatar Mathieu Giraud

Merge branch 'feature-s/more_stats_decorators' into 'dev'

Feature s/more stats decorators

See merge request !306
parents 92504489 8d2a6859
Pipeline #42589 passed with stages
in 6 minutes and 13 seconds
......@@ -1392,6 +1392,12 @@ span.warningReads {
background: inherit;
display: inline-block;
}
.table_stats td:hover {
z-index: 999;
position: relative;
background: inherit;
display: inline-block;
}
.db_table_split {
text-align: center;
background-color: #073642;
......@@ -2207,23 +2213,51 @@ form .form_label {
width: 85px;
}
.meter {
height: 5px;
height: 15px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #839496;
}
.meter > span {
.meter span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
}
.meter span.value_container span.meter_value {
background-color: #000000;
opacity: 0.5;
color: #839496;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
background-color: #839496;
background-image: linear-gradient(center bottom, #839496 37%, #000000 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
.bar_chart {
height: 20px;
}
.bar_chart .bar {
display: inline-block;
background-color: #839496;
background-image: linear-gradient(center bottom, #839496 37%, #000000 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
min-width: 0.2%;
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......@@ -1392,6 +1392,12 @@ span.warningReads {
background: inherit;
display: inline-block;
}
.table_stats td:hover {
z-index: 999;
position: relative;
background: inherit;
display: inline-block;
}
.db_table_split {
text-align: center;
background-color: #fdf6e3;
......@@ -2207,23 +2213,51 @@ form .form_label {
width: 85px;
}
.meter {
height: 5px;
height: 15px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
.meter span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
}
.meter span.value_container span.meter_value {
background-color: #ffffff;
opacity: 0.5;
color: #657b83;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
.bar_chart {
height: 20px;
}
.bar_chart .bar {
display: inline-block;
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
min-width: 0.2%;
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......@@ -1392,6 +1392,12 @@ span.warningReads {
background: inherit;
display: inline-block;
}
.table_stats td:hover {
z-index: 999;
position: relative;
background: inherit;
display: inline-block;
}
.db_table_split {
text-align: center;
background-color: #fdf6e3;
......@@ -2207,23 +2213,51 @@ form .form_label {
width: 85px;
}
.meter {
height: 5px;
height: 15px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
.meter span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
}
.meter span.value_container span.meter_value {
background-color: #ffffff;
opacity: 0.5;
color: #657b83;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
.bar_chart {
height: 20px;
}
.bar_chart .bar {
display: inline-block;
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
min-width: 0.2%;
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......
......@@ -1388,6 +1388,12 @@ span.warningReads {
background: inherit;
display: inline-block;
}
.table_stats td:hover {
z-index: 999;
position: relative;
background: inherit;
display: inline-block;
}
.db_table_split {
text-align: center;
background-color: #777777;
......@@ -2203,23 +2209,51 @@ form .form_label {
width: 85px;
}
.meter {
height: 5px;
height: 15px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #777777;
}
.meter > span {
.meter span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
}
.meter span.value_container span.meter_value {
background-color: #FFFFFF;
opacity: 0.5;
color: #777777;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
background-color: #777777;
background-image: linear-gradient(center bottom, #777777 37%, #FFFFFF 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
.bar_chart {
height: 20px;
}
.bar_chart .bar {
display: inline-block;
background-color: #777777;
background-image: linear-gradient(center bottom, #777777 37%, #FFFFFF 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
min-width: 0.2%;
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......
......@@ -1386,6 +1386,12 @@ span.warningReads {
background: inherit;
display: inline-block;
}
.table_stats td:hover {
z-index: 999;
position: relative;
background: inherit;
display: inline-block;
}
.db_table_split {
text-align: center;
background-color: #fdf6e3;
......@@ -2201,23 +2207,54 @@ form .form_label {
width: 85px;
}
.meter {
height: 5px;
height: 15px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
.meter span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
}
.meter span.value_container span.meter_value {
background-color: #ffffff;
opacity: 0.5;
color: #657b83;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
.bar_chart {
height: 20px;
}
.bar_chart .bar {
display: inline-block;
background-color: #657b83;
background-image: linear-gradient(center bottom, #657b83 37%, #ffffff 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
min-width: 0.2%;
}
.content.sample_sets {
font-size: 12px;
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......
......@@ -1581,6 +1581,10 @@ span.warningReads {
.ellipsisOverflow;
}
.table_stats td:hover {
.ellipsisOverflow;
}
.db_table_split{
text-align:center;
background-color: @highlight;
......@@ -2525,22 +2529,37 @@ form {
}
.meter {
height: 5px; /* Can be anything */
height: 15px; /* Can be anything */
position: relative;
background: #fff;
padding: 2px;
background: #e7e7e7;
box-shadow: inset 0 -1px 1px rgba(255,255,255,0.01);
border: 1px solid @default;
span.value_container {
width: 100%;
display: inline-block;
text-align: center;
vertical-align: middle;
z-index: 3;
margin-top: -20%;
span.meter_value {
background-color: @background;
opacity: 0.5;
color: @default;
height: 50%;
display: inline-block;
font-size: 10px;
text-align: center;
}
}
}
.meter > span {
.meter > span.meter_bar {
display: block;
height: 100%;
background-color: rgb(59, 219, 101);
background-color: @default;
background-image: linear-gradient(
center bottom,
rgb(59, 219, 101) 37%,
rgb(84,240,84) 69%
@default 37%,
@background 69%
);
box-shadow:
inset 0 1px 1px rgba(255,255,255,0.3),
......@@ -2549,6 +2568,27 @@ form {
overflow: hidden;
}
.bar_chart {
height: 20px;
.bar {
display: inline-block;
background-color: @default;
background-image: linear-gradient(
center bottom,
@default 37%,
@background 69%
);
box-shadow:
inset 0 1px 1px rgba(255,255,255,0.3),
inset 0 -1px 1px rgba(0,0,0,0.4);
min-width: 0.2%;
}
}
.stats_locus {
margin: 2px;
}
[hidden] {
display: none !important;
}
......@@ -158,7 +158,14 @@ def run_contamination():
log.error(res)
return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
def run_extra():
task = scheduler.queue_task('compute_extra', pvars=dict(id_file=request.vars["sequence_file_id"],
id_config=request.vars["config_id"],
min_threshold=5))
res = {"success" : "true",
"processId" : task.id}
log.debug(res)
return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
def checkProcess():
task = db.scheduler_task[request.vars["processId"]]
......
......@@ -5,6 +5,7 @@ import time
import json
from vidjilparser import VidjilParser
import operator
import math
if request.env.http_origin:
response.headers['Access-Control-Allow-Origin'] = request.env.http_origin
......@@ -657,22 +658,32 @@ def custom():
def getStatHeaders():
m = StatDecorator()
s = SetsDecorator()
b = BooleanDecorator()
p = BarDecorator()
return [('set_id', 'db', m), ('set_name', 'db', m), ('set_info', 'db', m), ('main_clone', 'parser', m), ('reads', 'parser', m), ('mapped', 'parser', m), ('mapped_percent', 'parser', p), ('bool', 'parser', b), ('bool_true', 'parser', b)]
def getResultsFileStats(file_name, dest):
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
with open(file_path, 'rb') as f:
objects = ijson.items(f, 'samples.results_file_id')
dest['results_file_ids'] = json.loads(mjson)['results_file_id']
return dest
bc = BarChartDecorator()
lbc = LabeledBarChartDecorator()
g = GenescanDecorator()
l = LociListDecorator()
return [('sample_sets', 'db', s),
#('reads', 'parser', m),
('mapped reads', 'parser', m),
#('mapped_percent', 'parser', p),
('read lengths', 'parser', g),
#('bool', 'parser', b),
#('bool_true', 'parser', b),
('loci', 'parser', l),
#('distribution', 'parser', lbc),
#('clones_five_percent', 'parser', m),
('main clone', 'parser', m)
#('abundance', 'parser', lbc)
]
def getFusedStats(file_name, res, dest):
log.debug("getFusedStats()")
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
parser = VidjilParser()
parser.addPrefix('clones.item', 'clones.item.top', operator.eq, 1)
parser.addPrefix('clones.item', 'clones.item.top', operator.le, 100)
parser.addPrefix("reads")
parser.addPrefix("samples")
......@@ -683,17 +694,70 @@ def getFusedStats(file_name, res, dest):
result_index = data['samples']['results_file_id'].index(res['resuts_file_id'])
elif "original_names" in data['samples']:
result_index = data['samples']['original_names'].index(defs.DIR_SEQUENCES + res['sequence_file'])
dest['main_clone'] = data['clones'][0]['name']
dest['main clone'] = data['clones'][0]['name']
reads = data['reads']['total'][result_index]
dest['reads'] = reads
dest['mapped'] = "%d" % (data['reads']['segmented'][result_index])
# dest['reads'] = reads
mapped_reads = data['reads']['segmented'][result_index]
dest['mapped reads'] = "%d / %d (%.2f %%)" % (mapped_reads, reads, 100.0*mapped_reads/reads if reads else 0)
dest['mapped_percent'] = 100.0 * (float(data['reads']['segmented'][result_index])/float(reads))
dest['bool'] = False
dest['bool_true'] = True
dest['abundance'] = [(key, 100.0*data['reads']['germline'][key][result_index]/reads) for key in data['reads']['germline']]
tmp = {}
for c in data['clones']:
try:
arl = int(math.ceil(c['_average_read_length'][result_index]))
except:
continue
if arl > 0:
if arl not in tmp:
tmp[arl] = 0.0
tmp[arl] += float(c['reads'][result_index])
min_len = 100 #int(min(tmp.keys()))
max_len = 600 #int(max(tmp.keys()))
tmp_list = []
if mapped_reads == 0:
mapped_reads = 1
for i in range(min_len, max_len):
if i in tmp:
if tmp[i]:
scaled_val = (2.5 + math.log10(tmp[i]/mapped_reads)) / 2
display_val = max(0.01, min(1, scaled_val)) * 100
else:
display_val = 0
real_val = 100.0*(tmp[i]/mapped_reads)
else:
display_val = 0
real_val = 0
tmp_list.append((i, display_val, real_val))
dest['read lengths'] = tmp_list
#dest['bool'] = False
#dest['bool_true'] = True
dest['loci'] = sorted([x for x in data['reads']['germline'] if data['reads']['germline'][x][result_index] > 0])
dest['clones_five_percent'] = sum([data['reads']['distribution'][key][result_index] for key in data['reads']['germline']])
return dest
def getResultsStats(file_name, dest):
import ijson.backends.yajl2_cffi as ijson
log.debug("getResultsStats()")
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
distributions = []
with open(file_path, 'rb') as results:
i = "1"
while True:
results.seek(0, 0)
tmp = [d for d in ijson.items(results, "reads-distribution-%s.item" % i)]
if len(tmp) == 0:
break
else:
distributions.append((i, tmp[0]))
i += "0"
dest['distribution'] = distributions
return dest
def getStatData(results_file_ids):
log.debug("getStatData(%s)" % str(results_file_ids))
mf = ModelFactory()
set_types = [defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN, defs.SET_TYPE_GENERIC]
helpers = {}
......@@ -703,17 +767,18 @@ def getStatData(results_file_ids):
query = db(
(db.results_file.id.belongs(results_file_ids)) &
(db.sequence_file.id == db.results_file.sequence_file_id) &
(db.config.id == db.results_file.config_id) &
(db.sample_set_membership.sequence_file_id == db.sequence_file.id) &
(db.sample_set.id == db.sample_set_membership.sample_set_id) &
(db.config.id == db.results_file.config_id) &
(db.fused_file.config_id == db.config.id) &
(db.fused_file.sample_set_id == db.sample_set.id)
(db.fused_file.sample_set_id == db.sample_set.id) &
(db.fused_file.config_id == db.config.id)
).select(
db.results_file.data_file.with_alias("results_file"), db.results_file.id.with_alias("results_file_id"),
db.fused_file.fused_file.with_alias("fused_file"),
db.results_file.sequence_file_id, db.results_file.config_id,
db.results_file.data_file.with_alias("data_file"), db.results_file.id.with_alias("results_file_id"),
db.sequence_file.data_file.with_alias("sequence_file"),
db.sample_set.id.with_alias("set_id"),
db.sample_set.sample_type.with_alias("sample_type"),
db.fused_file.fused_file.with_alias("fused_file"),
db.patient.first_name, db.patient.last_name, db.patient.info.with_alias('set_info'), db.patient.sample_set_id,
db.run.name,
db.generic.name,
......@@ -727,18 +792,41 @@ def getStatData(results_file_ids):
]
)
data = []
tmp_data = {}
for res in query:
set_type = res['sample_type']
if res.results_file_id not in tmp_data:
tmp = res.copy()
tmp['sample_sets'] = []
tmp_data[tmp['results_file_id']] = tmp
tmp.pop('set_id', None)
tmp.pop(set_type, None)
tmp.pop('set_info', None)
else:
tmp = tmp_data[res['results_file_id']]
sample_set = {}
sample_set['id'] = res['set_id']
sample_set['name'] = helpers[set_type].get_name(res[set_type])
sample_set['info'] = res['set_info']
sample_set['type'] = set_type
tmp['sample_sets'].append(sample_set)
data = []
for key in tmp_data:
res = tmp_data[key]
d = {}
set_type = res.sample_type
set_type = res['sample_type']
headers = getStatHeaders()
d = getFusedStats(res.fused_file, res, d)
d = getFusedStats(res['fused_file'], res, d)
#d = getResultsStats(res['data_file'], d)
for head, htype, model in headers:
if htype == 'db':
d[head] = res[head]
d[head] = model.decorate(d[head])
log.debug("%s: %s" % (head, d[head]))
d['set_name'] = helpers[set_type].get_name(res[set_type])
d['sequence_file_id'] = res['results_file']['sequence_file_id']
d['config_id'] = res['results_file']['config_id']
data.append(d)
return data
......@@ -762,8 +850,6 @@ def multi_sample_stats():
)
permitted_results_ids = [r.results_file_id for r in permitted_results]
log.debug("premitted: " + str(permitted_results_ids))
log.debug("custom: " + str(custom_result))
if set(permitted_results_ids) != set(custom_result):
res = {"message": ACCESS_DENIED}
log.error(res)
......
......@@ -26,4 +26,77 @@ class BarDecorator(StatDecorator):
super(BarDecorator, self).__init__()
def decorate(self, data):
return DIV(SPAN(_style="width: %d%%" % data), _class="meter", _title="%d%%" % data)
return DIV(SPAN(_class="meter_bar", _style="width: %d%%" % data), SPAN(SPAN("%d%%" % data, _class="meter_value"), _class="value_container"), _class="meter", _title="%d%%" % data)
class BarChartDecorator(StatDecorator):
def __init__(self):
super(BarChartDecorator, self).__init__()
def decorate(self, data):
bars = []
for val in data:
bar_span = SPAN(_style="height: %f%%; width: %f%%" % (val, (1.0/len(data))*100), _title="%f%%" % val, _class="bar")
bars.append(bar_span)
return DIV(*bars, _class="bar_chart")
class LabeledBarChartDecorator(BarChartDecorator):
def __init__(self):
super(LabeledBarChartDecorator, self).__init__()
def decorate(self, data):
bars = []
for t in data:
style = "height: %f%%; width: %f%%" % (t[1], (1.0/len(data))*100<