Commit 227f8e1b authored by Mathieu Giraud's avatar Mathieu Giraud
Browse files

Merge branch 'feature-sc/4722-nfs_pre_process' into 'prod-server'

nfs + pre_process

See merge request !971
parents 38f4833c ca3dc13e
Pipeline #256283 failed with stages
in 8 minutes and 37 seconds
......@@ -394,26 +394,6 @@ Database.prototype = {
}
});
},
pre_process_onChange : function (field) {
var $option = $(field).find(":selected");
if ($option.attr('required_files') == "1"){
$(".file_2").hide();
$(".upload_file").val("");
$(".upload_field").each(function() {
$(this).prop("required", false);
});
}else{
$(".file_2").show();
if ($(".is_editing").length == 0) {
// Not editing a sample, but creating new ones
$(".upload_field").each(function() {
$(this).prop("required", true);
});
}
}
},
upload_file_onChange : function (target_id, value) {
var target = document.getElementById(target_id);
......@@ -623,6 +603,12 @@ Database.prototype = {
$('#upload_form').on('submit', function(e) {
e.preventDefault();
self.update_upload_fields();
if (!self.check_upload_fields()) return;
$("#submit_samples_btn").addClass("disabledClass");
setTimeout(function(){$("#submit_samples_btn").removeClass("disabledClass")}, 3000)
//clear empty values before submiting data
var upload_form = $('#upload_form').serializeObject()
if ("file" in upload_form)
......@@ -714,54 +700,136 @@ Database.prototype = {
});
},
display_jstree: function(caller_index) {
$("#jstree_button").data("index", caller_index);
display_jstree: function(file_index, upload_index) {
$("#jstree_button").data("file_index", file_index);
$("#jstree_button").data("upload_index", upload_index);
$("#jstree_container").show();
$('#file_indicator_' + caller_index).text("");
$('#file_filename_' + caller_index).val("");
$('#file_indicator_' + file_index + "_" + upload_index).text("");
$('#file_filename_' + upload_index).val("");
},
close_jstree: function() {
$("#jstree_container").hide();
},
select_jstree: function(caller_index) {
$('#file_indicator_' + caller_index).text($('#file_indicator').text());
$('#file_filename_' + caller_index).val($('#file_filename').val());
select_jstree: function(file_index, upload_index) {
$('#file_indicator_' + file_index + "_" + upload_index).text($('#file_indicator').text());
$('#file_indicator_' + file_index + "_" + upload_index).prop('title', $('#file_filename').prop("value"));
$('#file_filename_' + file_index + "_" + upload_index).val($('#file_filename').val());
$("#jstree_container").hide();
},
toggle_upload_fields: function() {
var elem = $('.upload_field');
var disable = !elem.prop('disabled');
elem.prop('disabled', disable);
if (disable) {
elem.closest("div").hide();
elem.val(undefined);
$('.filename').val(undefined);
} else {
elem.closest("div").show();
check_upload_fields: function(){
file1 = $("[id^=file_filename_]");
file2 = $("[id^=file_filename2_]");
if ( $("#submitForm_isEditing").prop("checked")){
if (this.pprocess_required_file >1)
if ((file1[0].value == "" && file2[0].value != "") ||
(file1[0].value != "" && file2[0].value == "")){
console.log({"type": "flash",
"msg" : "missing file: both file fields must be filled if you wish to update current uploaded file.",
"priority": 2});
return false;
}
}else{
var flag = true;
for (var i=0; i<file1.length; i++)
if (file1[i].value == "" ) flag = false;
if (this.pprocess_required_file >1)
for (var j=0; j<file2.length; j++)
if (file2[j].value == "" ) flag = false;
if (!flag) {
console.log({"type": "flash",
"msg" : "missing file: please ensure all file fields are filled before submitting.",
"priority": 2});
return false
}
}
var pre_process = $('#pre_process');
pre_process.prop('disabled', disable);
pre_process.closest("div").prop('hidden', disable);
return true;
},
if (!disable) {
this.pre_process_onChange(pre_process);
update_upload_fields: function() {
//retrieve current radio buttons value
var radios = document.getElementsByName("source");
for (var i=0; i<radios.length; i++)
if (radios[i].checked)
this.upload_source = radios[i].value;
var option = $("#pre_process").find(":selected");
this.pprocess_required_file = parseInt(option.attr('required_files'))
// retrieve upload fields
var upload_fields = $('.upload_field');
var jstree_fields = $('.jstree_field');
// reset field, display/enable all upload field
jstree_fields.closest("div").show();
jstree_fields.prop("disabled", false);
upload_fields.closest("div").show();
upload_fields.prop("disabled", false);
// hide/disabe unnecessary field for selected upload source
if (this.upload_source == "nfs"){
upload_fields.closest("div").hide();
upload_fields.prop("disabled", true);
}
if (this.upload_source == "computer"){
jstree_fields.closest("div").hide();
jstree_fields.prop("disabled", true);
}
// hide/disable unnecessary field for selected pre-process
if (this.pprocess_required_file == 1){
upload_fields.filter('.file_2').closest("div").hide();
upload_fields.filter('.file_2').prop("disabled", true);
jstree_fields.closest("div").filter('.file_2').hide();
jstree_fields.filter('.file_2').prop("disabled", true);
}
this.update_hidden_fields();
this.update_jstree();
},
update_hidden_fields:function(){
//reset default filename
var forms = $('.form_line')
for (var i=0; i<forms.length; i++){
var filename="";
var filename2="";
if (this.upload_source == "computer"){
filename = $(forms[i]).find(".upload_field.file_1")[0].value;
var lastIndex = filename.lastIndexOf('\\');
if (lastIndex > 0) filename = filename.substring(lastIndex + 1);
filename2 = $(forms[i]).find(".upload_field.file_2")[0].value;
var lastIndex2 = filename2.lastIndexOf('\\');
if (lastIndex2 > 0) filename2 = filename2.substring(lastIndex2 + 1);
}
if (this.upload_source == "nfs"){
filename = $(forms[i]).find("[id^=file_indicator_1]").prop('title');
filename2 = $(forms[i]).find("[id^=file_indicator_2]").prop('title');
}
$(forms[i]).find("[id^=file_filename_]")[0].value = filename;
$(forms[i]).find("[id^=file_filename2_]")[0].value = filename2;
}
},
toggle_jstree: function(){
update_jstree: function(){
var tree = $('.jstree_field');
var enable = tree.prop('hidden');
var enable = this.upload_source == "nfs";
tree.prop('hidden', !enable);
},
toggle_file_source: function() {
this.toggle_upload_fields();
this.toggle_jstree()
},
/**
* reload the current db page
......
......@@ -454,7 +454,7 @@ FileFormBuilder.prototype = {
build_file_div: function() {
var self = this;
var hide_second = this.source || this.num_files < 2;
var hide_second = this.num_files < 2;
var d = document.createElement('div');
d.className="field_div";
var file1 = this.build_file_field(1, this.source);
......@@ -464,7 +464,8 @@ FileFormBuilder.prototype = {
}
d.appendChild(file1);
d.appendChild(this.build_file_field(2, hide_second));
d.appendChild(this.build_jstree(!this.source));
d.appendChild(this.build_jstree(1, !this.source));
d.appendChild(this.build_jstree(2, hide_second));
return d;
},
......@@ -475,6 +476,10 @@ FileFormBuilder.prototype = {
i.hidden = true;
i.className = '';
d.appendChild(i);
i = this.build_input('filename2', 'filename2', 'filename2', 'text', 'file');
i.hidden = true;
i.className = '';
d.appendChild(i);
i = this.build_input('id', '', 'id', 'text', 'file');
i.hidden = true;
i.className = '';
......@@ -524,12 +529,13 @@ FileFormBuilder.prototype = {
},
build_file_field: function(id, hidden) {
var self = this;
var d = this.build_wrapper();
d.className += " file_" + id;
d.id = "file_field_" + id + "_" + self.index;
if (this.source || hidden) {
d.style.display = "none";
}
var i = this.build_input('upload_' + id, 'upload_field', 'file'+id, 'file', 'file');
var i = this.build_input('upload_' + id, 'upload_field file_'+ id, 'file'+id, 'file', 'file');
if (this.source) {
i.disabled = true;
} else if (! hidden) {
......@@ -540,19 +546,19 @@ FileFormBuilder.prototype = {
return d;
},
build_jstree: function() {
build_jstree: function(id, hidden) {
var self = this;
var w = this.build_wrapper();
var d = document.createElement('div');
d.id = "jstree_field_" + self.index;
w.id = "jstree_field_" + id + "_" + self.index;
w.appendChild(d);
d.className += " jstree_field form-control";
if (!this.source) {
d.hidden = true;
d.className += " jstree_field form-control file_" + id;
if (!this.source || hidden) {
d.style.display = "none";
}
d.onclick = function() {
db.display_jstree(self.index);
db.display_jstree(id, self.index);
}
var sel = document.createElement('span');
......@@ -560,9 +566,9 @@ FileFormBuilder.prototype = {
sel.appendChild(document.createTextNode(('browse')));
d.appendChild(sel);
var indicator = document.createElement('span');
indicator.id = "file_indicator_" + self.index;
indicator.id = "file_indicator_" + id + "_" + self.index;
d.appendChild(indicator);
return d;
return w;
}
}
FileFormBuilder.prototype = $.extend(Object.create(FormBuilder.prototype), FileFormBuilder.prototype)
......@@ -56,10 +56,29 @@ def link_to_sample_sets(seq_file_id, id_dict):
db.commit()
# TODO put these in a model or utils or smth
def validate(myfile):
def validate(myfile, id , pre_process):
reupload = id != ""
error = []
if myfile['filename'] == None :
error.append("missing filename")
# 0 or two filename must be provided to update a file with pre-process
if reupload and pre_process is not None:
if myfile['filename'] == "" and myfile['filename2'] != "" :
error.append("missing filename")
if myfile['filename2'] == "" and myfile['filename'] != "" :
error.append("missing filename2")
# both filename must be provided to add a file with pre-process
if not reupload and pre_process is not None:
if myfile['filename'] == "" :
error.append("missing filename")
if myfile['filename2'] == "":
error.append("missing filename2")
# a single filename must be provided to add a file without pre_process
if not reupload and pre_process is None :
if myfile['filename'] == "" :
error.append("missing filename")
if myfile['sampling_date'] != '' :
try:
datetime.datetime.strptime(""+myfile['sampling_date'], '%Y-%m-%d')
......@@ -220,7 +239,7 @@ def submit():
error = True
for f in data['file']:
f['errors'] = validate(f)
f['errors'] = validate(f, f["id"], pre_process)
f['sets'], f['id_dict'], err = validate_sets(f['set_ids'])
......@@ -279,16 +298,29 @@ def submit():
mes += " file was replaced"
file_data, filepath = manage_filename(f["filename"])
filename = file_data['filename']
if 'data_file' in file_data and file_data['data_file'] is not None:
os.symlink(filepath, defs.DIR_SEQUENCES + file_data['data_file'])
file_data['size_file'] = os.path.getsize(filepath)
file_data['network'] = True
file_data['data_file'] = str(file_data['data_file'])
if data["source"] == "nfs" :
file_data2, filepath2 = manage_filename(f["filename2"])
if 'data_file' in file_data2 and file_data2['data_file'] is not None:
file_data['data_file2'] = str(file_data2['data_file'])
os.symlink(filepath2, defs.DIR_SEQUENCES + file_data2['data_file'])
db.sequence_file[fid] = file_data
link_to_sample_sets(fid, id_dict)
# pre-process for nfs files can be started immediately
data_file = db.sequence_file[fid].data_file
data_file2 = db.sequence_file[fid].data_file2
if data["source"] == "nfs" :
if data_file is not None and data_file2 is not None and pre_process != '0':
schedule_pre_process(fid, pre_process)
log.info(mes, extra={'user_id': auth.user.id,
'record_id': f['id'],
'table_name': "sequence_file"})
......@@ -302,7 +334,7 @@ def submit():
"message": "successfully added/edited file(s)"}
return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
else:
return form_response(data)
return error_message("add_form() failed")
def form_response(data):
source_module_active = hasattr(defs, 'FILE_SOURCE') and hasattr(defs, 'FILE_TYPES')
......@@ -365,7 +397,7 @@ def upload():
db.sequence_file[request.vars["id"]] = dict(pre_process_flag=None,
pre_process_result=None)
if data_file is not None and data_file2 is not None and request.vars['pre_process'] != '0':
if data_file is not None and data_file2 is not None and request.vars['pre_process'] is not None and request.vars['pre_process'] != '0':
db.sequence_file[request.vars["id"]] = dict(pre_process_flag = "WAIT")
old_task_id = db.sequence_file[request.vars["id"]].pre_process_scheduler_task_id
if db.scheduler_task[old_task_id] != None:
......
......@@ -72,7 +72,7 @@ class TestSample < ServerTest
jstree = $b.div(:id => "jstree")
for i in 0..$num_additional_files
$b.div(:id => "jstree_field_%d" % i).span(:text => "browse").click
$b.div(:id => "jstree_field_1_%d" % i).span(:text => "browse").click
assert(jstree.present?)
jstree_file = jstree.a(:id => "//Demo-X5.fa_anchor")
unless jstree_file.present? and jstree_file.present?
......@@ -89,13 +89,10 @@ class TestSample < ServerTest
# TODO test other sets
end
form_class = form.input(:type => "submit").class_name
assert( !(form_class.include? "disabledClass") ) # Submit button is not disabled before click
assert( !form.input(:type => "submit").obscured? ) # Submit button is not disabled before click
form.input(:type => "submit").click
form_class = form.input(:type => "submit").class_name
assert( (form_class.include? "disabledClass") ) # Submit button should be disabled after click
assert( form.input(:type => "submit").obscured? ) # Submit button should be disabled after click
table = $b.table(:id => "table")
table.wait_until(&:present?)
......@@ -254,7 +251,7 @@ class TestSample < ServerTest
$b.input(:id => "source_nfs").click
jstree = $b.div(:id => "jstree")
$b.div(:id => "jstree_field_0").span(:text => "browse").click
$b.div(:id => "jstree_field_1_0").span(:text => "browse").click
assert(jstree.present?)
jstree_file = jstree.a(:id => "//Demo-X5.fa_anchor")
unless jstree_file.present? and jstree_file.present?
......
......@@ -68,6 +68,7 @@ class FileController(unittest.TestCase):
data = {}
data['set_ids'] = ":p plapipou ("+str(fake_patient_id)+")"
data['source'] = "computer"
myfile= {
"id": "",
......@@ -77,6 +78,7 @@ class FileController(unittest.TestCase):
"sequencer": "plop",
"producer": "plop",
"filename": "plopapi",
"filename2": "",
"set_ids": ""
}
data['file'] = [myfile]
......@@ -84,8 +86,53 @@ class FileController(unittest.TestCase):
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertNotEqual(resp.find('"redirect":"sample_set/index"'), -1, "add_form() failed")
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# nfs test
defs.FILE_SOURCE = os.path.join(os.path.dirname(__file__), '../../../../../demo/')
data['source'] = "nfs"
myfile['filename'] = "//Demo-X5.fa"
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# invalid form 1
defs.FILE_SOURCE = os.path.join(os.path.dirname(__file__), '../../../../../demo/')
data['source'] = "nfs"
myfile['filename'] = ""
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"add_form() failed"'), 0, "edit_form() was expected to fail but succeed"+resp)
# nfs + pre-process
data['source'] = "nfs"
data['pre_process'] = str(fake_pre_process_id)
myfile['filename'] = "//Demo-X5.fa"
myfile['filename2'] = "//Demo-X5.fa"
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# invalid form 2
data['source'] = "nfs"
data['pre_process'] = str(fake_pre_process_id)
myfile['filename'] = "//Demo-X5.fa"
myfile['filename2'] = ""
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"add_form() failed"'), 0, "edit_form() was expected to fail but succeed"+resp)
def testEdit(self):
request.vars['sample_set_id'] = fake_sample_set_id
......@@ -99,10 +146,12 @@ class FileController(unittest.TestCase):
data = {}
data['set_ids']=":p plapipou ("+str(fake_patient_id)+")"
data["sample_type"] = defs.SET_TYPE_PATIENT
data['source'] = "computer"
myfile = {
"id" : fake_file_id,
"filename" : "plopapi",
"filename2": "",
"sampling_date" : "1992-02-02",
"info" : "plop",
"pcr" : "plop",
......@@ -113,11 +162,56 @@ class FileController(unittest.TestCase):
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertEqual(resp.find('"message":"plopapi: metadata saved"'), -1, "edit_form() failed")
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# nfs test
defs.FILE_SOURCE = os.path.join(os.path.dirname(__file__), '../../../../../demo/')
data['source'] = "nfs"
myfile['filename'] = "//Demo-X5.fa"
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# nfs + pre-process
data['source'] = "nfs"
data['pre_process'] = str(fake_pre_process_id)
myfile['filename'] = "//Demo-X5.fa"
myfile['filename2'] = "//Demo-X5.fa"
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
# invalid form
data['source'] = "nfs"
data['pre_process'] = str(fake_pre_process_id)
myfile['filename'] = ""
myfile['filename2'] = "//Demo-X5.fa"
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"add_form() failed"'), 0, "edit_form() was expected to fail but succeed"+resp)
# edit back to defaultsource computer
data['source'] = "computer"
data['pre_process'] = "0"
myfile['filename'] = "plopapi"
myfile['filename2'] = ""
data['file'] = [myfile]
request.vars['data'] = json.dumps(data)
resp = submit()
self.assertGreater(resp.find('"message":"successfully added/edited file(s)"'), 0, "edit_form() failed"+resp)
def testUpload(self):
class emptyClass( object ):
......
......@@ -10,16 +10,17 @@
{{ required_files = 1 }}
{{ if source_module_active: }}
<span>file source: </span>
<input hidden type=checkbox id="submitForm_isEditing" name="isEditing" {{ if isEditing: }} checked {{ pass }}>
<label for="source_computer">my computer</label>
<input type="radio" id="source_computer" name="source" value="computer" onchange="db.toggle_file_source()" {{ if not network_source: }} checked="checked" {{pass}}/>
<input type="radio" id="source_computer" name="source" value="computer" onchange="db.update_upload_fields()" {{ if not network_source: }} checked="checked" {{pass}}/>
<label for="source_nfs">network</label>
<input type="radio" id="source_nfs" name="source" value="nfs" onchange="db.toggle_file_source()" {{if network_source:}} checked="checked" {{pass}} />
<input type="radio" id="source_nfs" name="source" value="nfs" onchange="db.update_upload_fields()" {{if network_source:}} checked="checked" {{pass}} />
<p></p>
{{pass}}
<div {{if network_source:}} hidden {{pass}}>
<div>
<h3>Pre-process scenario</h3>
If you have two R1/R2 files per sample, please select an appropriate pre-process :
<select id="pre_process" name="pre_process" onChange="db.pre_process_onChange(this)" {{if network_source:}} disabled {{pass}}>
<select id="pre_process" name="pre_process" onChange="db.update_upload_fields()">