Commit 1ae6a8e8 authored by Ryan Herbert's avatar Ryan Herbert

Merge branch 'feature-s/3171-stats-multi-samples' into 'dev'

Feature s/3171 stats multi samples

See merge request !269
parents 27ffaf74 d507f798
Pipeline #36598 failed with stages
in 50 minutes and 20 seconds
......@@ -2197,6 +2197,24 @@ form .form_label {
min-width: 85px;
width: 85px;
}
.meter {
height: 5px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #839496;
}
.meter > span {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......@@ -2197,6 +2197,24 @@ form .form_label {
min-width: 85px;
width: 85px;
}
.meter {
height: 5px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......@@ -2197,6 +2197,24 @@ form .form_label {
min-width: 85px;
width: 85px;
}
.meter {
height: 5px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......
......@@ -2196,6 +2196,24 @@ form .form_label {
min-width: 85px;
width: 85px;
}
.meter {
height: 5px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #777777;
}
.meter > span {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......
......@@ -2191,6 +2191,24 @@ form .form_label {
min-width: 85px;
width: 85px;
}
.meter {
height: 5px;
/* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255, 255, 255, 0.01);
border: 1px solid #657b83;
}
.meter > span {
display: block;
height: 100%;
background-color: #3bdb65;
background-image: linear-gradient(center bottom, #3bdb65 37%, #54f054 69%);
box-shadow: inset 0 1px 1px rgba(255, 255, 255, 0.3), inset 0 -1px 1px rgba(0, 0, 0, 0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......
......@@ -2513,6 +2513,32 @@ form {
}
}
.meter {
height: 5px; /* Can be anything */
position: relative;
background: #fff;
padding: 2px;
box-shadow: inset 0 -1px 1px rgba(255,255,255,0.01);
border: 1px solid @default;
}
.meter > span {
display: block;
height: 100%;
background-color: rgb(59, 219, 101);
background-image: linear-gradient(
center bottom,
rgb(59, 219, 101) 37%,
rgb(84,240,84) 69%
);
box-shadow:
inset 0 1px 1px rgba(255,255,255,0.3),
inset 0 -1px 1px rgba(0,0,0,0.4);
position: relative;
overflow: hidden;
}
[hidden] {
display: none !important;
}
......@@ -181,10 +181,12 @@ These instructions are preliminary, other documentation can also be found in [[h
apt-get install g++
apt-get install make
apt-get install unzip
apt-get install python-pip
apt-get install python-dev python-pip
apt-get install libyajl2 libyajl-dev
pip install unittest2
pip install unittest-xml-reporting
pip install enum34
pip install ijson cffi
#+END_SRC
** Vidjil server installation and initialization
......
......@@ -6,7 +6,7 @@ with a full installation of the Vidjil algoright and browser/server."
env GOSU_VERSION 1.7
run set -x \
&& apt-get update && apt-get install -y --no-install-recommends ca-certificates wget cron unzip make python ipython python-enum34 python-requests git && rm -rf /var/lib/apt/lists/* \
&& apt-get update && apt-get install -y --no-install-recommends ca-certificates wget cron unzip make python ipython python-enum34 python-requests git python-ijson libyajl2 python-cffi && rm -rf /var/lib/apt/lists/* \
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
&& export GNUPGHOME="$(mktemp -d)" \
......
......@@ -6,3 +6,5 @@ requests==2.12.3
six==1.11.0
unittest-xml-reporting==2.1.1
uWSGI==2.0.14
ijson==2.3
cffi==1.11.5
......@@ -3,6 +3,8 @@ import gluon.contrib.simplejson, datetime
import vidjil_utils
import time
import json
from vidjilparser import VidjilParser
import operator
if request.env.http_origin:
response.headers['Access-Control-Allow-Origin'] = request.env.http_origin
......@@ -651,6 +653,124 @@ def custom():
tag_decorator=tag_decorator,
group_ids=group_ids)
def getStatHeaders():
m = StatDecorator()
b = BooleanDecorator()
p = BarDecorator()
return [('set_id', 'db', m), ('set_name', 'db', m), ('set_info', 'db', m), ('main_clone', 'parser', m), ('reads', 'parser', m), ('mapped', 'parser', m), ('mapped_percent', 'parser', p), ('bool', 'parser', b), ('bool_true', 'parser', b)]
def getResultsFileStats(file_name, dest):
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
with open(file_path, 'rb') as f:
objects = ijson.items(f, 'samples.results_file_id')
dest['results_file_ids'] = json.loads(mjson)['results_file_id']
return dest
def getFusedStats(file_name, res, dest):
file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
parser = VidjilParser()
parser.addPrefix('clones.item', 'clones.item.top', operator.eq, 1)
parser.addPrefix("reads")
parser.addPrefix("samples")
mjson = parser.extract(file_path)
data = json.loads(mjson)
result_index = -1
if "results_file_id" in data['samples']:
result_index = data['samples']['results_file_id'].index(res['resuts_file_id'])
elif "original_names" in data['samples']:
result_index = data['samples']['original_names'].index(defs.DIR_SEQUENCES + res['sequence_file'])
dest['main_clone'] = data['clones'][0]['name']
reads = data['reads']['total'][result_index]
dest['reads'] = reads
dest['mapped'] = "%d" % (data['reads']['segmented'][result_index])
dest['mapped_percent'] = 100.0 * (float(data['reads']['segmented'][result_index])/float(reads))
dest['bool'] = False
dest['bool_true'] = True
return dest
def getStatData(results_file_ids):
mf = ModelFactory()
set_types = [defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN, defs.SET_TYPE_GENERIC]
helpers = {}
for stype in set_types:
helpers[stype] = mf.get_instance(stype)
query = db(
(db.results_file.id.belongs(results_file_ids)) &
(db.sequence_file.id == db.results_file.sequence_file_id) &
(db.sample_set_membership.sequence_file_id == db.sequence_file.id) &
(db.sample_set.id == db.sample_set_membership.sample_set_id) &
(db.config.id == db.results_file.config_id) &
(db.fused_file.config_id == db.config.id) &
(db.fused_file.sample_set_id == db.sample_set.id)
).select(
db.results_file.data_file.with_alias("results_file"), db.results_file.id.with_alias("results_file_id"),
db.fused_file.fused_file.with_alias("fused_file"),
db.sequence_file.data_file.with_alias("sequence_file"),
db.sample_set.id.with_alias("set_id"),
db.sample_set.sample_type.with_alias("sample_type"),
db.patient.first_name, db.patient.last_name, db.patient.info.with_alias('set_info'), db.patient.sample_set_id,
db.run.name,
db.generic.name,
db.config.name,
db.generic.name.with_alias("set_name"), # use generic name as failsafe for set name
left = [
db.patient.on(db.patient.sample_set_id == db.sample_set.id),
db.run.on(db.run.sample_set_id == db.sample_set.id),
db.generic.on(db.generic.sample_set_id == db.sample_set.id)
]
)
data = []
for res in query:
d = {}
set_type = res.sample_type
headers = getStatHeaders()
d = getFusedStats(res.fused_file, res, d)
for head, htype, model in headers:
if htype == 'db':
d[head] = res[head]
d[head] = model.decorate(d[head])
log.debug("%s: %s" % (head, d[head]))
d['set_name'] = helpers[set_type].get_name(res[set_type])
data.append(d)
return data
def multi_sample_stats():
data = {}
data['headers'] = [h for h, t, m in getStatHeaders()]
results = []
custom_result = request.vars['custom_result']
if not isinstance(custom_result, list):
custom_result = [custom_result]
custom_result = [long(i) for i in custom_result]
permitted_results = db(
(auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set)) &
(db.sample_set.id == db.sample_set_membership.sample_set_id) &
(db.sample_set_membership.sequence_file_id == db.results_file.sequence_file_id) &
(db.results_file.id.belongs(custom_result))
).select(
db.results_file.id.with_alias('results_file_id')
)
permitted_results_ids = [r.results_file_id for r in permitted_results]
log.debug("premitted: " + str(permitted_results_ids))
log.debug("custom: " + str(custom_result))
if set(permitted_results_ids) != set(custom_result):
res = {"message": ACCESS_DENIED}
log.error(res)
return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
results = getStatData(custom_result)
data['results'] = results
return dict(data=data)
def confirm():
if auth.can_modify_sample_set(request.vars["id"]):
sample_set = db.sample_set[request.vars["id"]]
......
import vidjil_utils
class StatDecorator(object):
def __init__(self):
pass
def decorate(self, data):
return data if data is not None else ""
class BooleanDecorator(StatDecorator):
def __init__(self):
super(BooleanDecorator, self).__init__()
def decorate(self, data):
if data:
myclass = "icon-ok"
else:
myclass = "icon-cancel"
return I(_class=myclass)
class BarDecorator(StatDecorator):
def __init__(self):
super(BarDecorator, self).__init__()
def decorate(self, data):
return DIV(SPAN(_style="width: %d%%" % data), _class="meter", _title="%d%%" % data)
#!/usr/bin/python
import ijson.backends.yajl2_cffi as ijson
from six import string_types
from enum import Enum
class MatchingEvent(Enum):
end_map = "start_map"
end_array = "start_array"
class VidjilWriter(object):
def __init__(self, pretty=False):
self.pretty = pretty
self.buffer = []
self.buffering = False
self.conserveBuffer = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def write(self, prefix, event, value, previous):
res = self._write(prefix, event, value, previous)
if self.buffering:
self.buffer.append(res)
return ""
return res
def _write(self, prefix, event, value, previous):
if self.pretty:
end = '\n'
else:
end = ''
if event == 'start_map':
mstr = '{{'
elif event == 'end_map':
mstr = '}}'
elif event == 'start_array':
mstr = '['
elif event == 'end_array':
mstr = ']'
elif event == 'map_key':
mstr = '\"{}\":'
end = ''
elif event == 'string':
mstr = '\"{}\"'
else:
if event == 'boolean':
value = str(value).lower()
mstr = '{}'
padding = ''
if isinstance(value, string_types) :
value = value.replace("\n", "\\n")
value = value.replace("\r", "\\r")
if previous not in ['', 'map_key', 'start_map', 'start_array'] and event not in ['end_map', 'end_array']:
mstr = "," + mstr
if self.pretty and previous != 'map_key':
if len(prefix) > 0:
padding = ''.join(['\t' for i in range(len(prefix.split('.')))])
mstr = '{}' + mstr + end
return mstr.format(padding, value)
def purgeBuffer(self):
self.buffer = []
self.conserveBuffer = False
def writeBuffer(self):
try:
return ''.join(self.buffer)
finally:
self.purgeBuffer()
def startBuffering(self):
self.conserveBuffer = False
self.buffering = True
def endBuffering(self):
self.buffering = False
if self.conserveBuffer:
self.conserveBuffer = False
return self.writeBuffer()
else :
self.purgeBuffer()
return ""
class VidjilFileWriter(VidjilWriter):
def __init__(self, filepath=None, pretty=False):
super(VidjilWriter, self).__init__()
self._filepath = filepath
self.file = None
def __enter__(self):
self.file = open(self._filepath, 'wb')
return self
def __exit__(self, exc_type, exc_value, traceback):
self.file.close()
def write(self, prefix, event, value, previous):
res = super(VidjilWriter, self).write(prefix, event, value, previous)
self.file.write(res)
return res
def writeBuffer(self):
res = super(VidjilWriter, self).writeBuffer()
self.file.write(res)
return res
class Predicate(object):
def __init__(self, field, comparator, value):
self.comp = comparator
self.field = field
self.value = value
def compare(self, field, other):
if self.comp is None:
return field == self.field
try:
res = field == self.field and self.comp(other, self.value)
return res
except:
return False
class VidjilParser(object):
def __init__(self, writer=None):
if writer is not None:
self._writer = writer
else:
self._writer = VidjilWriter()
self._model_prefixes = []
self.prefixes = []
def initModel(self, model_path):
with open(model_path, 'rb') as model:
parser = ijson.parse(model)
for prefix, event, value in parser:
if (prefix, event) not in self._model_prefixes:
self._model_prefixes.append((prefix, event))
def validate(self, filepath):
with open(filepath, 'rb') as vfile:
parser = ijson.parse(vfile)
model = list(self._model_prefixes)
for prefix, event, value in parser:
pair = (prefix, event)
if pair in model:
model.remove(pair)
return len(model) == 0
def writer(self):
return self._writer
def addPrefix(self, prefix, conditional = None, comp = None, value = None):
if conditional is None:
conditional = prefix
self.prefixes.append((prefix, Predicate(conditional, comp, value)))
def reset(self):
self.prefixes = []
self._writer.purgeBuffer()
def extract(self, filepath):
vidjilfile = open(filepath, 'rb')
parser = ijson.parse(vidjilfile)
with self.writer() as writer:
return self._extract(parser, writer)
def isStartEvent(self, event):
return event in ['start_map', 'start_array']
def isEndEvent(self, event):
return event in ['end_map', 'end_array', 'number', 'string', 'boolean']
def isMatching(self, mbuffer, other):
if other[1] not in MatchingEvent.__members__:
return False
return (mbuffer[0] == other[0]) and (mbuffer[1] == MatchingEvent[other[1]].value)
def _extract(self, parser, writer):
previous = ''
res = ""
bufferStart = (None, None)
for prefix, event, value in parser:
subelem = lambda x, y: x.startswith(y)
if any(subelem(prefix, item[0])\
or (subelem(item[0], prefix) and (value is None or subelem(item[0], str(value))))\
for item in self.prefixes):
bufferOn = any(prefix == item[0] for item in self.prefixes) and self.isStartEvent(event)
if bufferOn:
bufferStart = (prefix, event)
saved_previous = previous
self._writer.startBuffering()
if not self._writer.conserveBuffer \
and any((item[1].compare(prefix, value)) for item in self.prefixes):
self._writer.conserveBuffer = True
res += writer.write(prefix, event, value, previous)
previous = event
if (self.writer().buffering and (self.isEndEvent(event) and self.isMatching(bufferStart, (prefix, event)) or self._writer.conserveBuffer)):
if not self._writer.conserveBuffer:
previous = saved_previous
res += self._writer.endBuffering()
return res
......@@ -103,6 +103,7 @@
<div class="db_block_right">
<span class="button2" onclick="myUrl.loadCustomUrl(db)" > see results </span>
<span class="devel-mode button2" onclick="db.call('sample_set/multi_sample_stats', {'custom_result': db.getListInput('custom_result[]') })" >stats</span>
</div>
</div>
......
{{extend 'db_layout.html'}}
<div id="db_table_container">
<table class="db_table" id="table">
<thead>
<tr>
{{ for h in data['headers']: }}
<td class="{{=h}}">
<span class="toggle {{=h}}"onclick="$('.content.{{=h}}').toggle(function() {if ($('.content.{{=h}}').is(':visible')) {$('.{{=h}}').width(''); $('.toggle.{{=h}}').text('[-]')} else {$('.{{=h}}').width('20px'); $('.toggle.{{=h}}').text('[+]')}})">[-]</span>
<span class="content {{=h}}">{{=h}}</span>
</td>
{{ pass }}
</tr>
</thead>
<tbody>
{{ for res in data['results']: }}
<tr>
{{ for h in data['headers']: }}
<td class="{{=h}}">
<span class="content {{=h}}">{{=res[h]}}</span>
</td>
{{ pass }}
</tr>
{{ pass }}
</tbody>
</table>
<table class="db_table" id="db_fixed_header"></table>
</div>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment