Commit 525bf1ad authored by Ryan Herbert's avatar Ryan Herbert

Merge alternative file sources

Allows user to select files from a local directory or mounted directory
instead of uploading files
parents 76332b41 2bcfb05c
......@@ -1279,6 +1279,7 @@ span.warningReads {
.db_msg {
margin: 15px;
height: calc(100% - 30px);
overflow-y: auto;
}
#db_content {
height: calc(100% - 160px);
......@@ -1823,3 +1824,12 @@ label.highlight_label {
.db_table td.pre-line {
white-space: pre-line;
}
#jstree {
max-height: 300px;
overflow-y: auto;
}
.jstree-anchor {
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -1286,6 +1286,7 @@ span.warningReads {
.db_msg {
margin: 15px;
height: calc(100% - 30px);
overflow-y: auto;
}
#db_content {
height: calc(100% - 160px);
......@@ -1831,3 +1832,12 @@ label.highlight_label {
.db_table td.pre-line {
white-space: pre-line;
}
#jstree {
max-height: 300px;
overflow-y: auto;
}
.jstree-anchor {
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
......@@ -1285,6 +1285,7 @@ span.warningReads {
.db_msg {
margin: 15px;
height: calc(100% - 30px);
overflow-y: auto;
}
#db_content {
height: calc(100% - 160px);
......@@ -1830,6 +1831,15 @@ label.highlight_label {
.db_table td.pre-line {
white-space: pre-line;
}
#jstree {
max-height: 300px;
overflow-y: auto;
}
.jstree-anchor {
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
#resolution5 {
opacity: 0;
}
......
......@@ -1286,6 +1286,7 @@ span.warningReads {
.db_msg {
margin: 15px;
height: calc(100% - 30px);
overflow-y: auto;
}
#db_content {
height: calc(100% - 160px);
......@@ -1831,6 +1832,15 @@ label.highlight_label {
.db_table td.pre-line {
white-space: pre-line;
}
#jstree {
max-height: 300px;
overflow-y: auto;
}
.jstree-anchor {
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
#mid-container {
font-size: 125%;
}
......
......@@ -1459,7 +1459,8 @@ span.warningReads {
.db_msg{
margin : 15px;
height : calc(~"100% - 30px")
height : calc(~"100% - 30px");
overflow-y: auto;
}
#db_content{
......@@ -2069,3 +2070,14 @@ label.highlight_label {
.db_table td.pre-line {
white-space: pre-line;
}
#jstree {
max-height: 300px;
overflow-y: auto;
}
.jstree-anchor {
max-width: 200px;
overflow: hidden;
text-overflow: ellipsis;
}
......@@ -32,6 +32,7 @@
<link rel="stylesheet" type="text/css" href="css/icons/fontello/css/animation.css" />
<link id="palette" rel="stylesheet" type="text/css" href="css/light.css" />
<link id="seg_highlight" rel="stylesheet" type="text/css" href="css/segmenter_highlight.css" />
<link rel="stylesheet" href="css/jstree_themes/default/style.min.css" />
<script data-main="js/app.js" src="js/lib/require.js"></script>
......
......@@ -14,7 +14,8 @@ require(["jquery",
"underscore",
"rgbcolor",
"file",
"tsne"], function() {
"tsne",
"jstree.min"], function() {
// Then config file (needed by Vidjil)
require(['../conf'], function() {
loadAfterConf()
......
......@@ -498,6 +498,59 @@ Database.prototype = {
}
},
set_jstree: function(elem) {
elem.jstree({
"plugins" : ["sort"],
'core' : {
'multiple': false,
'data' : {
'url' : function(node){
var address = DB_ADDRESS + '/file/filesystem'
return node.id === '#' ? address
: address + '?node=' + node.id
},
'dataType' : 'json',
},
}
});
elem.on('select_node.jstree', function(event, data){
$('#filename').val(data.selected);
});
},
toggle_upload_fields: function() {
var elem = $('.upload_field');
var disable = !elem.prop('disabled');
elem.prop('disabled', disable);
elem.parents("tr").prop('hidden', disable);
if (disable) {
elem.val(undefined);
$('#filename').val(undefined);
}
var pre_process = $('#pre_process');
pre_process.prop('disabled', disable);
pre_process.parents("tr").prop('hidden', disable);
},
toggle_jstree: function(){
var tree = $('#jstree');
var enable = tree.parents("tr").prop('hidden');
tree.parents("tr").prop('hidden', !enable);
if (enable) {
this.set_jstree(tree);
} else {
$.jstree.destroy();
tree.off('select_node.jstree');
$('#filename').val(undefined);
}
},
toggle_file_source: function() {
this.toggle_upload_fields();
this.toggle_jstree()
},
/**
* reload the current db page
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -66,3 +66,7 @@ LOCUS = ['TRA', 'TRA+D', 'TRB', 'TRG', 'TRD', 'TRD+',
FS_LOCK_THRESHHOLD = 1
SCHEDULER_HEARTBEAT = 5
# Directory to search for files
FILE_SOURCE = '/mnt/data/src'
FILE_TYPES = ['fasta', 'fastq', 'fastq.gz']
......@@ -6,6 +6,8 @@ import os
import os.path
import datetime
from controller_utils import error_message
import jstree
import base64
if request.env.http_origin:
......@@ -116,6 +118,7 @@ def add():
run = run_date+name+id
source_module_active = hasattr(defs, 'FILE_SOURCE') and hasattr(defs, 'FILE_TYPES')
return dict(message = T('add file'),
generic_list = generic_list,
patient_list = patient_list,
......@@ -124,8 +127,27 @@ def add():
generic = generic,
patient = patient,
sample_type = sample_set.sample_type,
run = run)
run = run,
source_module_active = source_module_active)
def manage_filename(filename):
filepath = ""
name_list = []
name_list = request.vars['filename'].split('/')
filename = name_list[-1]
data = dict(filename=filename, data_file=None)
if len(name_list) > 1:
filepath = defs.FILE_SOURCE + '/' + request.vars['filename']
split_file = filename.split('.')
uuid_key = db.uuid().replace('-', '')[-16:]
encoded_filename = base64.b16encode('.'.join(split_file[0:-1])).lower()
data_file = "sf.%s.%s.%s" % (
uuid_key, encoded_filename, split_file[-1]
)
data['data_file'] = data_file
return (data, filepath)
def add_form():
error = ""
......@@ -141,6 +163,10 @@ def add_form():
if request.vars['filename'] == None :
error += " missing file"
else:
data, filepath = manage_filename(request.vars["filename"])
filename = data['filename']
if request.vars['patient_id'] == '' and request.vars['run_id'] == "" and request.vars['generic_id'] == "":
error += " missing patient or run or sample_set"
......@@ -157,8 +183,9 @@ def add_form():
&(db.sequence_file.id == db.sample_set_membership.sequence_file_id)
).select(db.sequence_file.ALL)
for row in query :
if row.filename == request.vars['filename'] :
if row.filename == filename :
error += " this sequence file already exists for this patient"
break
if request.vars['run_id'] != '' :
try:
......@@ -177,19 +204,26 @@ def add_form():
error += " invalid sample_set %s" % request.vars['sample_set_id']
pre_process = None
pre_process_flag = "DONE"
if request.vars['pre_process'] != "0":
if request.vars['pre_process'] is not None and request.vars['pre_process'] != "0":
pre_process = request.vars['pre_process']
pre_process_flag = "WAIT"
if error=="" :
#add sequence_file to the db
id = db.sequence_file.insert(sampling_date=request.vars['sampling_date'],
info=request.vars['file_info'],
filename=request.vars['filename'],
pre_process_id=pre_process,
pre_process_flag=pre_process_flag,
provider=auth.user_id)
log_message = "upload started"
if request.vars['filename'] != "":
if data['data_file'] is not None:
log_message = "registered"
os.symlink(filepath, defs.DIR_SEQUENCES + data['data_file'])
db.sequence_file[id] = data
ids_sample_set = []
#add sequence_file to a run sample_set
if run_id is not None :
......@@ -221,7 +255,7 @@ def add_form():
res = {"file_id" : id,
"message": "(%s) file {%s} : upload started: %s" % (','.join(map(str,ids_sample_set)), id, request.vars['filename']),
"message": "(%s) file {%s} : %s: %s" % (','.join(map(str,ids_sample_set)), id, log_message, request.vars['filename']),
"redirect": "sample_set/index",
"args" : redirect_args
}
......@@ -375,14 +409,19 @@ def edit_form():
# file is being reuploaded, remove all existing results_files
db(db.results_file.sequence_file_id == request.vars["id"]).delete()
pre_process = None
if request.vars['pre_process'] != "0":
if request.vars['pre_process'] is not None and request.vars['pre_process'] != "0":
pre_process = int(request.vars['pre_process'])
if request.vars['sampling_date'] != None and request.vars['file_info'] != None :
db.sequence_file[request.vars["id"]] = dict(sampling_date=request.vars['sampling_date'],
info=request.vars['file_info'],
filename=filename,
pre_process_id=pre_process,
provider=auth.user_id)
if request.vars['filename'] != "":
data, filepath = manage_filename(request.vars["filename"])
if 'data_file' in data:
os.symlink(filepath, defs.DIR_SEQUENCES + data['data_file'])
db.sequence_file[request.vars["id"]] = data
#remove previous membership
for row in db( db.sample_set_membership.sequence_file_id == request.vars["id"]).select() :
......@@ -597,3 +636,21 @@ def restart_pre_process():
db.commit()
res = schedule_pre_process(sequence_file.id, pre_process.id)
return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
def filesystem():
json = []
id = "" if request.vars["node"] is None else request.vars["node"] + '/'
if id == "":
json = [{"text": "/", "id": "/", "children": True}]
else:
root_folder = defs.FILE_SOURCE + id
for idx, f in enumerate(os.listdir(root_folder)):
correct_type = f.split('.')[-1] in defs.FILE_TYPES
is_dir = os.path.isdir(root_folder + f)
if correct_type or is_dir:
json_node = jstree.Node(f, id + f).jsonData()
if is_dir : json_node['children'] = True
if correct_type: json_node['icon'] = 'jstree-file'
json_node['li_attr']['title'] = f
json.append(json_node)
return gluon.contrib.simplejson.dumps(json, separators=(',',':'))
......@@ -71,3 +71,7 @@ LOCUS = ['TRA', 'TRA+D', 'TRB', 'TRG', 'TRD', 'TRD+',
FS_LOCK_THRESHHOLD = 1
SCHEDULER_HEARTBEAT = 5
# Directory to search for files
FILE_SOURCE = '/mnt/data/src'
FILE_TYPES = ['fasta', 'fastq', 'fastq.gz']
import pickle
class DictionaryObject(object):
"""
A class that has all the functionality of a normal Python dictionary, except
for the fact it is itself immutable. It also has the added feature of
being able to lookup values by using keys as attributes.
The reason for the class being immutable by default is to help make it a
little easier to use in multiprocessing situations. Granted, the underlying
values themselves are not deeply copied, but the aim is to enforce some
ensurances of immutability on the container class.
When using positional arguments, the first argument must always be something
that would be a valid argument for a dict(). However, a second, optional
argument may be passed to create a default value when keys are not found.
Examples:
>>> d = DictionaryObject({'a':1, 'b':True, 3:'x'})
>>> d.a == 1
True
>>> d.b
True
>>> d[3] == 'x'
True
>>> d = DictionaryObject((('a',1),('b',2)))
>>> d.a == 1
True
>>> d.b == 2
True
>>> d = DictionaryObject({'a':1, 'b':True}, None)
>>> d.a == 1
True
>>> d.b
True
>>> d.c
>>> d = DictionaryObject({'a':1}, None)
>>> m = MutableDictionaryObject(d)
>>> d == m
True
>>> m.a = 0
>>> d == m
False
>>> d != m
True
>>> import pickle
>>> m1 = MutableDictionaryObject({'a':1}, None)
>>> m2 = pickle.loads(pickle.dumps(m1))
>>> m1 == m2
True
>>> m1.a = 3
>>> m1 == m2
False
>>> m1.a == 3
True
>>> m1['c'] = 5
>>> m1['c']
5
"""
def __init__(self, contents=(), *args, **kwargs):
"""
Take as input a dictionary-like object and return a DictionaryObject.
It also makes sure any keys containing dictionaries are also converted
to DictionaryObjects.
"""
super(DictionaryObject, self).__init__()
if isinstance(contents, DictionaryObject):
self.__dict__.update(pickle.loads(pickle.dumps(contents.__dict__)))
return
self.__dict__['_items'] = dict(contents, **kwargs)
if len(args) > 1:
raise TypeError("too many arguments")
# If we have more than one argument passed in, use the second argument
# as a default value.
if args:
try:
default = type(self)(args[0])
except:
default = args[0]
self.__dict__['_defaultValue'] = default
else:
self.__dict__['_defaultValue'] = None
self.__dict__['_defaultIsSet'] = len(args) > 0
for k in self._items:
if isinstance(self._items[k], dict):
self._items[k] = type(self)(self._items[k])
def __setstate__(self, dict):
self.__dict__.update(dict)
def __getstate__(self):
return self.__dict__.copy()
def __getattr__(self, name):
"""
This is the method that makes all the magic happen. Search for
'name' in self._items and return the value if found. If a default
value has been set and 'name' is not found in self._items, return it.
Otherwise raise an AttributeError.
Example:
>>> d = DictionaryObject({'keys':[1,2], 'values':3, 'x':1})
>>> sorted(list(d.keys())) == ['keys', 'values', 'x']
True
>>> [1, 2] in list(d.values())
True
>>> 1 in list(d.values())
True
>>> d.x
1
>>> d['keys']
[1, 2]
>>> d['values']
3
"""
if name in self._items:
return self._items[name]
if self._defaultIsSet:
return self._defaultValue
raise AttributeError("'%s' object has no attribute '%s'" % (type(self).__name__, name))
def __setattr__(self, name, value):
"""
This class is immutable-by-default. See MutableDictionaryObject.
"""
raise AttributeError("'%s' object does not support assignment" % type(self).__name__)
def __getitem__(self, name):
return self._items[name]
def __contains__(self, name):
return name in self._items
def __len__(self):
return len(self._items)
def __iter__(self):
return iter(self._items)
def __repr__(self):
if self._defaultIsSet:
params = "%s, %s" % (repr(self._items), self._defaultValue)
else:
params = repr(self._items)
return "%s(%s)" % (type(self).__name__, params)
def __cmp__(self, rhs):
if self < rhs:
return -1
if self > rhs:
return 1
return 0
def __eq__(self, rhs):
if self._items == rhs._items:
return self._defaultValue == rhs._defaultValue
return False
def __ne__(self, rhs):
return not (self == rhs)
def keys(self):
return self._items.keys()
def values(self):
return self._items.values()
def asdict(self):
"""
Copy the data back out of here and into a dict. Then return it.
Some libraries may check specifically for dict objects, such
as the json library; so, this makes it convenient to get the data
back out.
>>> import dictobj
>>> d = {'a':1, 'b':2}
>>> dictobj.DictionaryObject(d).asdict() == d
True
>>> d['c'] = {1:2, 3:4}
>>> dictobj.DictionaryObject(d).asdict() == d
True
"""
items = {}
for name in self._items:
value = self._items[name]
if isinstance(value, DictionaryObject):
items[name] = value.asdict()
else:
items[name] = value
return items
class MutableDictionaryObject(DictionaryObject):
"""
Slight enhancement of the DictionaryObject allowing one to add
attributes easily, in cases where that functionality is wanted.
Examples:
>>> d = MutableDictionaryObject({'a':1, 'b':True}, None)
>>> d.a == 1
True
>>> d.b == True
True
>>> d.c is None
True
>>> d.d is None
True
>>> d.c = 3
>>> del d.a
>>> d.a is None
True
>>> d.c == 3
True
"""
def __setattr__(self, name, value):
self._items[name] = value
def __delattr__(self, name):
del self._items[name]
__setitem__ = __setattr__
__delitem__ = __delattr__
import dictobj
import os
from collections import namedtuple
Path = namedtuple('Path', ('path', 'id'))
class Node(dictobj.DictionaryObject):
"""
This class exists as a helper to the jsTree. Its "jsonData" method can
generate sub-tree JSON without putting the logic directly into the jsTree.
This data structure is only semi-immutable. The jsTree uses a directly
iterative (i.e. no stack is managed) builder pattern to construct a
tree out of paths. Therefore, the children are not known in advance, and
we have to keep the children attribute mutable.
"""
def __init__(self, path, oid, **kwargs):
"""
kwargs allows users to pass arbitrary information into a Node that
will later be output in jsonData(). It allows for more advanced
configuration than the default path handling that jsTree currently allows.
For example, users may want to pass "attr" or some other valid jsTree options.
Example:
>>> from dictobj import *
>>> node = Node('a', None)
>>> node._items == {'text': 'a', 'children': MutableDictionaryObject({})}
True
>>> node.jsonData() == {'text': 'a'}
True
>>> import jstree
>>> from dictobj import *
>>> node = jstree.Node('a', 1)
>>> node._items == {'text': 'a', 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 1})}
True
>>> d = node.jsonData()
>>> d == {'text': 'a', 'li_attr': {'id': 1}}
True
>>> import jstree
>>> node = jstree.Node('a', 5, icon="folder", state = {'opened': True})
>>> node._items == {'text': 'a', 'state': DictionaryObject({'opened': True}), 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 5}), 'icon': 'folder'}
True
>>> node.jsonData() == {'text': 'a', 'state': {'opened': True}, 'li_attr': {'id': 5}, 'icon': 'folder'}
True
"""
super(Node, self).__init__()
children = kwargs.get('children', {})
</