file.py 19.1 KB
Newer Older
Marc Duez's avatar
Marc Duez committed
1
# coding: utf8
Marc Duez's avatar
Marc Duez committed
2
import gluon.contrib.simplejson
Marc Duez's avatar
Marc Duez committed
3
import defs
4
import vidjil_utils
Marc Duez's avatar
Marc Duez committed
5
import os
6 7
import os.path
import datetime
8
from controller_utils import error_message
Ryan Herbert's avatar
Ryan Herbert committed
9
import jstree
10
import base64
11

12

Marc Duez's avatar
Marc Duez committed
13 14 15 16 17
if request.env.http_origin:
    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin  
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400

18
def extract_set_type(target):
Ryan Herbert's avatar
Ryan Herbert committed
19 20 21 22 23
    mapping = {
        'p': 'patient',
        'r': 'run',
        's': 'generic'
    }
24
    return mapping[target.split(':')[1][0]]
25

Ryan Herbert's avatar
Ryan Herbert committed
26

27
def manage_filename(filename):
28 29
    filepath = ""
    name_list = []
Ryan Herbert's avatar
Ryan Herbert committed
30 31 32
    name_list = filename.split('/')
    myfilename = name_list[-1]
    data = dict(filename=myfilename, data_file=None)
33 34

    if len(name_list) > 1:
Ryan Herbert's avatar
Ryan Herbert committed
35 36
        filepath = defs.FILE_SOURCE + '/' + filename
        split_file = myfilename.split('.')
37 38
        uuid_key = db.uuid().replace('-', '')[-16:]
        encoded_filename = base64.b16encode('.'.join(split_file[0:-1])).lower()
39
        data_file = "sequence_file.data_file.%s.%s.%s" % (
40 41
                uuid_key, encoded_filename, split_file[-1]
            )
42 43 44
        data['data_file'] = data_file

    return (data, filepath)
45

46 47 48 49 50 51
def link_to_sample_sets(seq_file_id, id_dict):
    '''
    Create sample set memberships and return a dict of the sample set ids.
    The keys to the dict are thee same as the ones passed in id_dict
    '''
    for key in id_dict:
52 53
        arr = [{'sample_set_id': oid, 'sequence_file_id': seq_file_id} for oid in id_dict[key]]
        db.sample_set_membership.bulk_insert(arr)
54
    db.commit()
55

56 57 58 59 60 61 62 63 64 65 66 67
# TODO put these in a model or utils or smth
def validate(myfile):
    error = []
    if myfile['filename'] == None :
        error.append("missing filename")
    if myfile['sampling_date'] != '' :
        try:
            datetime.datetime.strptime(""+myfile['sampling_date'], '%Y-%m-%d')
        except ValueError:
            error.append("date (wrong format)")
    return error

68 69 70 71
def validate_sets(set_ids):
    id_dict = {}
    sets = []
    errors = []
72

73
    if len(set_ids) == 0:
74 75 76 77 78
        errors.append("missing set association")

    mf = ModelFactory()
    helpers = {}

79 80
    set_ids_arr = []
    if len(set_ids) > 0:
81
        set_ids_arr = [x.strip() for x in set_ids.split('|')]
82
    for sid in set_ids_arr:
83 84
        try:
            set_type = extract_set_type(sid)
85
            if set_type not in id_dict:
86
                helpers[set_type] = mf.get_instance(set_type)
87 88
                id_dict[set_type] = []
            sets.append({'type': set_type, 'id': sid})
89
            set_id = helpers[set_type].parse_id_string(sid)
90
            id_dict[set_type].append(set_id)
91
            if not auth.can_modify_sample_set(set_id) :
92 93
                errors.append("missing permission for %s %d" % (set_type, set_id))
        except ValueError:
Ryan Herbert's avatar
Ryan Herbert committed
94
            errors.append("Invalid %s %s" % (set_type, sid))
95
    return sets, id_dict, errors
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116

def get_pre_process_list():
    query_pre_process = db(
        db.pre_process>0
    ).select(
        db.pre_process.ALL,
        orderby = ~db.pre_process.id
    )

    pre_process_list = []
    for row in query_pre_process :
        file = 1
        if "&file2" in row.command:
                file = 2
        pre_process_list.append(dict(
                        id = row.id,
                        name = row.name,
                        file = file,
                        info = row.info
                ))
    return pre_process_list
117

118 119 120 121 122 123 124 125
def get_set_list(id_dict, helpers):
    sets = []
    for key in id_dict:
        slist = db(db[key].id.belongs(id_dict[key])).select()
        for sset in slist:
            sets.append({'type': key, 'id': helpers[key].get_id_string(sset)})
    return sets

126
def get_set_helpers():
Ryan Herbert's avatar
Ryan Herbert committed
127 128 129 130 131
    factory = ModelFactory()
    sample_types = [defs.SET_TYPE_GENERIC, defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN]
    helpers = {}
    for stype in sample_types:
        helpers[stype] = factory.get_instance(type=stype)
132 133 134 135 136 137 138
    return helpers

def form():
    group_ids = []
    relevant_ids = {}

    helpers = get_set_helpers()
Ryan Herbert's avatar
Ryan Herbert committed
139 140 141 142 143 144

    # new file
    if 'sample_set_id' in request.vars:
        sample_set = db.sample_set[request.vars["sample_set_id"]]
        if not auth.can_upload_sample_set(sample_set.id):
            return error_message("you don't have right to upload files")
145

Ryan Herbert's avatar
Ryan Herbert committed
146 147 148 149 150 151 152
        sample_type = sample_set.sample_type
        enough_space = vidjil_utils.check_enough_space(defs.DIR_SEQUENCES)
        if not enough_space:
            mail.send(to=defs.ADMIN_EMAILS,
                subject="[Vidjil] Server space",
                message="The space in directory %s has passed below %d%%." % (defs.DIR_SEQUENCES, defs.FS_LOCK_THRESHHOLD))
            return error_message("Uploads are temporarily disabled. System admins have been made aware of the situation.")
153

Ryan Herbert's avatar
Ryan Herbert committed
154
        row = db(db[sample_set.sample_type].sample_set_id == request.vars["sample_set_id"]).select().first()
Ryan Herbert's avatar
Ryan Herbert committed
155 156 157 158
        stype = sample_set.sample_type
        if stype not in relevant_ids:
            relevant_ids[stype] = []
        relevant_ids[stype].append(row.id)
159
        action = 'add'
160

Ryan Herbert's avatar
Ryan Herbert committed
161 162 163 164
    # edit file
    elif 'file_id' in request.vars:
        if not auth.can_modify_file(request.vars['file_id']):
            return error_message("you need admin permission to edit files")
HERBERT Ryan's avatar
HERBERT Ryan committed
165 166

        sample_set_list = db(
Ryan Herbert's avatar
Ryan Herbert committed
167
                (db.sample_set_membership.sequence_file_id == request.vars['file_id'])
HERBERT Ryan's avatar
HERBERT Ryan committed
168
                & (db.sample_set_membership.sample_set_id != None)
169 170
                & (db.sample_set.id == db.sample_set_membership.sample_set_id)
                & (db.sample_set.sample_type != 'sequence_file')
HERBERT Ryan's avatar
HERBERT Ryan committed
171
            ).select(db.sample_set_membership.sample_set_id)
172
        for row in sample_set_list :
Ryan Herbert's avatar
Ryan Herbert committed
173
            smp_type = db.sample_set[row.sample_set_id].sample_type
Ryan Herbert's avatar
Ryan Herbert committed
174 175 176
            if smp_type not in relevant_ids:
                relevant_ids[smp_type] = []
            relevant_ids[smp_type].append(db(db[smp_type].sample_set_id == row.sample_set_id).select()[0].id)
177
        action = 'edit'
178

Ryan Herbert's avatar
Ryan Herbert committed
179
        sample_type = request.vars["sample_type"]
180
    else:
Ryan Herbert's avatar
Ryan Herbert committed
181 182
        return error_message("missing sample_set or file id")

183 184 185
    myfile = db.sequence_file[request.vars["file_id"]]
    if myfile is None:
        myfile = {}
186
    myfile['sets'] = []
187
    sets = get_set_list(relevant_ids, helpers)
Ryan Herbert's avatar
Ryan Herbert committed
188

189 190 191 192 193
    data = {}
    data['file'] = [myfile]
    data['sets'] = sets
    data['sample_type'] = sample_type
    data['errors'] = []
194
    data['action'] = action
Ryan Herbert's avatar
Ryan Herbert committed
195

196
    return form_response(data)
197 198

#TODO check data
Ryan Herbert's avatar
Ryan Herbert committed
199
def submit():
200 201
    data = json.loads(request.vars['data'], encoding='utf-8')
    error = False
202 203 204 205 206 207 208 209

    pre_process = None
    pre_process_flag = "DONE"
    if request.vars['pre_process'] is not None:
        if f['pre_process'] != "0":
            pre_process = int(f['pre_process'])
            pre_process_flag = "WAIT"

210 211 212 213 214 215
    sets, id_dict, errors = validate_sets(data['set_ids'])
    data['sets'] = sets

    data['errors'] = errors

    if len(errors) > 0:
216
        error = True
217

218
    for f in data['file']:
219
        f['errors'] = validate(f)
220

221 222
        f['sets'], f['id_dict'], err = validate_sets(f['set_ids'])

223
        if len(f['errors']) > 0:
224 225 226 227 228
            error = True
            continue

        file_data = dict(sampling_date=f['sampling_date'],
                         info=f['info'],
229
                         pre_process_id=pre_process,
230 231 232 233 234 235 236 237 238 239 240
                         pre_process_flag= pre_process_flag,
                         provider=auth.user_id)

        # edit
        if (f["id"] != ""):
            reupload = True
            fid = int(f["id"])
            sequence_file = db.sequence_file[fid]
            db.sequence_file[fid] = file_data
            #remove previous membership
            db( db.sample_set_membership.sequence_file_id == fid).delete()
241
            action = "edit"
242 243 244 245

        # add
        else:
            reupload = False
246
            f['id'] = fid = db.sequence_file.insert(**file_data)
247
            action = "add"
248

249
        data['action'] = action
250
        f['message'] = []
251
        mes = "file (%d) %s %sed" % (int(f["id"]), f["filename"], action)
Ryan Herbert's avatar
Ryan Herbert committed
252 253
        f['message'].append(mes)
        f['message'].append("You must reselect the file for it to be uploaded")
254 255 256 257 258 259

        for key in f['id_dict']:
            if key not in id_dict:
                id_dict[key] = []
            id_dict[key] += f['id_dict'][key]

260 261
        for key in id_dict:
            for sid in id_dict[key]:
262 263
                group_id = get_set_group(sid)
                register_tags(db, 'sequence_file', fid, f["info"], group_id, reset=True)
264 265 266 267 268

        if f['filename'] != "":
            if reupload:
                # file is being reuploaded, remove all existing results_files
                db(db.results_file.sequence_file_id == fid).delete()
269
                mes += " file was replaced"
270 271 272 273 274 275 276 277 278

            file_data, filepath = manage_filename(f["filename"])
            filename = file_data['filename']
            if 'data_file' in file_data and file_data['data_file'] is not None:
                os.symlink(filepath, defs.DIR_SEQUENCES + file_data['data_file'])
                file_data['size_file'] = os.path.getsize(filepath)
                file_data['network'] = True
            db.sequence_file[fid] = file_data

279
        link_to_sample_sets(fid, id_dict)
280

281
        log.info(mes, extra={'user_id': auth.user.id,\
282
                'record_id': f['id'],\
283
                'table_name': "sequence_file"})
Ryan Herbert's avatar
Ryan Herbert committed
284

285
    if not error:
286
        set_id = data['sets'][0]['id']
287
        res = { "file_ids": [f['id'] for f in data['file']],
288 289
                "redirect": "sample_set/index",
                "args" : { "id" : set_id, "config_id": -1},
290 291
                "message": "successfully added/edited file(s)"}
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
292
    else:
293
        return form_response(data)
294
    
295 296 297 298 299 300 301 302 303 304 305 306 307 308
def form_response(data):
    source_module_active = hasattr(defs, 'FILE_SOURCE') and hasattr(defs, 'FILE_TYPES')
    response.view = 'file/form.html'
    upload_group_ids = [int(gid) for gid in get_upload_group_ids(auth)]
    group_ids = [int(g['id']) for g in get_default_creation_group(auth)[0]]
    pre_process_list = get_pre_process_list()
    return dict(message=T("an error occured"),
           pre_process_list = pre_process_list,
           files = data['file'],
           sets = data['sets'],
           sample_type = data['sample_type'],
           errors = data['errors'],
           source_module_active = source_module_active,
           group_ids = group_ids,
309 310
           upload_group_ids = upload_group_ids,
           isEditing = data['action']=='edit')
311

312
def upload(): 
Marc Duez's avatar
Marc Duez committed
313
    session.forget(response)
314
    mes = ""
315
    error = ""
316

317 318
    if request.vars['id'] == None :
        error += "missing id"
319 320
    elif db.sequence_file[request.vars["id"]] is None:
        error += "no sequence file with this id"
321

322
    if not error:
323
        mes += " file {%s} " % (request.vars['id'])
324
        res = {"message": mes + "processing uploaded file"}
325
        log.debug(res)
326
        if request.vars.file != None :
Marc Duez's avatar
Marc Duez committed
327
            f = request.vars.file
328
            try:
329 330 331 332
                if request.vars["file_number"] == "1" :
                    db.sequence_file[request.vars["id"]] = dict(data_file = db.sequence_file.data_file.store(f.file, f.filename))
                else :
                    db.sequence_file[request.vars["id"]] = dict(data_file2 = db.sequence_file.data_file.store(f.file, f.filename))
333
                mes += "upload finished (%s)" % (f.filename)
334 335
            except IOError as e:
                if str(e).find("File name too long") > -1:
Mikaël Salson's avatar
Mikaël Salson committed
336
                    error += 'Your filename is too long, please shorten it.'
337
                else:
Mikaël Salson's avatar
Mikaël Salson committed
338
                    error += "System error during processing of uploaded file."
339
                    log.error(str(e))
340
        
341
        data_file = db.sequence_file[request.vars["id"]].data_file
342 343 344
        data_file2 = db.sequence_file[request.vars["id"]].data_file2
        
        if request.vars["file_number"] == "1" and len(error) == 0 and data_file is None:
345
            error += "no data file"
346 347
        if request.vars["file_number"] == "2" and len(error) == 0 and data_file2 is None:
            error += "no data file"
348 349 350

        db.sequence_file[request.vars["id"]] = dict(pre_process_flag=None,
                                                    pre_process_result=None)
351
        if data_file is not None and data_file2 is not None and request.vars['pre_process'] != '0':
352
            db.sequence_file[request.vars["id"]] = dict(pre_process_flag = "WAIT")
353 354 355 356 357
            old_task_id = db.sequence_file[request.vars["id"]].pre_process_scheduler_task_id
            if db.scheduler_task[old_task_id] != None:
                scheduler.stop_task(old_task_id)
                db(db.scheduler_task.id == old_task_id).delete()
                db.commit()
358
            schedule_pre_process(int(request.vars['id']), int(request.vars['pre_process']))
359
            mes += " | p%s start pre_process %s " % (request.vars['pre_process'], request.vars['id'] + "-" +request.vars['pre_process'])
360

361 362 363 364 365 366
        if data_file is not None :
            seq_file = defs.DIR_SEQUENCES + data_file
            # Compute and store file size
            size = os.path.getsize(seq_file)
            mes += ' (%s)' % vidjil_utils.format_size(size)
            db.sequence_file[request.vars["id"]] = dict(size_file = size)
367

368 369 370 371
        if data_file2 is not None :
            seq_file2 = defs.DIR_SEQUENCES + data_file2
            #TODO
        
372
    # Log and exit
Mikaël Salson's avatar
Mikaël Salson committed
373
    res = {"message": error + mes}
374
    if error:
Mikaël Salson's avatar
Mikaël Salson committed
375
        res['success'] = 'false'
376
        res['priority'] = 3
377 378
        log.error(res)
    else:
Mathieu Giraud's avatar
Mathieu Giraud committed
379
        log.info(res)
380
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Marc Duez's avatar
Marc Duez committed
381
  
382

383
def confirm():
384 385 386 387 388
    '''
    Request parameters:
    \param delete_results: (optional) boolean
    \param id: sequence file ID
    '''
389 390 391 392
    delete_only_sequence = ('delete_only_sequence' in request.vars and request.vars['delete_only_sequence'] == 'True')
    delete_results = ('delete_results' in request.vars and request.vars['delete_results'] == 'True')
    sequence_file = db.sequence_file[request.vars['id']]
    if sequence_file == None:
393
        return error_message("The requested file doesn't exist")
394 395
    if sequence_file.data_file == None:
        delete_results = True
396
    if auth.can_modify_sample_set(request.vars['redirect_sample_set_id']):
397 398 399
        return dict(message=T('choose what you would like to delete'),
                    delete_only_sequence = delete_only_sequence,
                    delete_results = delete_results)
400
    else:
401
        return error_message("you need admin permission to delete this file")
402

403 404 405
def delete_sequence_file(seq_id):
    sequence = db.sequence_file[seq_id]
    seq_filename = sequence.data_file
406 407

    if auth.can_modify_file(seq_id):
408 409
        if seq_filename is not None:
            log.debug('Deleting '+defs.DIR_SEQUENCES+seq_filename+' with ID'+str(seq_id))
410 411 412 413
        db.sequence_file[seq_id] = dict(data_file = None)
    else:
        return error_message('you need admin permission to delete this file')

414
def delete():
415 416 417 418 419
    '''
    Called (via request) with:
    \param: id (the sequence ID)
    \param: delete_results: (optional) boolean stating if we also want to delete the results.
    '''
420
    delete_results = ('delete_results' in request.vars and request.vars['delete_results'] == "True")
421 422
    sample_set = db.sample_set[request.vars["redirect_sample_set_id"]]
    associated_id = None
423
    if sample_set.sample_type not in ['sequence_file', 'sample_set']:
424
        associated_elements = db(db[sample_set.sample_type].sample_set_id == sample_set.id).select()
425
        if len(associated_elements) > 0:
426
            associated_id = associated_elements[0].id
427

428
    if auth.can_modify_file(request.vars["id"]):
429 430 431
        if not(delete_results):
            delete_sequence_file(request.vars['id'])
        else:
432 433
            sample_set_ids = get_sequence_file_sample_sets(request.vars["id"])
            config_ids = get_sequence_file_config_ids(request.vars["id"])
434
            db(db.results_file.sequence_file_id == request.vars["id"]).delete()
435
            db(db.sequence_file.id == request.vars["id"]).delete()
436
            schedule_fuse(sample_set_ids, config_ids)
437

438 439 440
        res = {"redirect": "sample_set/index",
               "args" : { "id" : request.vars["redirect_sample_set_id"]},
               "message": "sequence file deleted"}
441
        if associated_id is not None:
442
            log.info(res, extra={'user_id': auth.user.id, 'record_id': associated_id, 'table_name': sample_set.sample_type})
443 444
        else:
            log.info(res)
445 446
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    else:
447
        return error_message("you need admin permission to delete this file")
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474

def sequencer_list():
    sequencer_list = []
    for row in db(db.sequence_file.sequencer != None).select(db.sequence_file.sequencer, distinct=True):
        if row.sequencer is not "null" :
            sequencer_list.append(row.sequencer)
            
    res = {"sequencer": sequencer_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def pcr_list():
    pcr_list = []
    for row in db(db.sequence_file.pcr != None).select(db.sequence_file.pcr, distinct=True):
        if row.pcr is not "null" :
            pcr_list.append(row.pcr)
            
    res = {"pcr": pcr_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def producer_list():
    producer_list = []
    for row in db(db.sequence_file.producer != None).select(db.sequence_file.producer, distinct=True):
        if row.producer is not "null" :
            producer_list.append(row.producer)
            
    res = {"producer": producer_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
475 476

def restart_pre_process():
477
    if "sequence_file_id" not in request.vars:
478 479 480 481
        return error_message("missing parameter")
    sequence_file = db.sequence_file[request.vars["sequence_file_id"]]
    if sequence_file is None or not auth.can_modify_file(sequence_file.id):
        return error_message("Permission denied")
482
    pre_process = db.pre_process[sequence_file.pre_process_id]
483 484
    db.sequence_file[sequence_file.id] = dict(pre_process_flag = 'WAIT')
    db.commit()
485 486
    res = schedule_pre_process(sequence_file.id, pre_process.id)
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
487

488 489
def match_filetype(filename, extension):
    ext_len = len(extension)
Mikaël Salson's avatar
Mikaël Salson committed
490
    return ext_len == 0 or filename[-ext_len:] == extension
491

Ryan Herbert's avatar
Ryan Herbert committed
492
def filesystem():
493
    json = []
494 495 496 497 498 499
    id = "" if request.vars["node"] is None else request.vars["node"] + '/'
    if id == "":
        json = [{"text": "/", "id": "/",  "children": True}]
    else:
        root_folder = defs.FILE_SOURCE + id
        for idx, f in enumerate(os.listdir(root_folder)):
Ryan Herbert's avatar
Ryan Herbert committed
500 501 502 503 504
            correct_type = False
            for ext in defs.FILE_TYPES:
                correct_type = match_filetype(f, ext)
                if correct_type:
                    break
505 506 507 508 509
            is_dir = os.path.isdir(root_folder + f)
            if correct_type or is_dir:
                json_node = jstree.Node(f, id + f).jsonData()
                if is_dir : json_node['children'] = True
                if correct_type: json_node['icon'] = 'jstree-file'
510
                json_node['li_attr']['title'] = f
511 512
                json.append(json_node)
    return gluon.contrib.simplejson.dumps(json, separators=(',',':'))