file.py 20.5 KB
Newer Older
Marc Duez's avatar
Marc Duez committed
1
# coding: utf8
Marc Duez's avatar
Marc Duez committed
2
import gluon.contrib.simplejson
Marc Duez's avatar
Marc Duez committed
3
import defs
4
import vidjil_utils
Marc Duez's avatar
Marc Duez committed
5
import os
6 7
import os.path
import datetime
8
from controller_utils import error_message
Ryan Herbert's avatar
Ryan Herbert committed
9
import jstree
10
import base64
11

12

Marc Duez's avatar
Marc Duez committed
13 14 15 16 17
if request.env.http_origin:
    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin  
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400

18
def extract_set_type(target):
Ryan Herbert's avatar
Ryan Herbert committed
19 20 21 22 23
    mapping = {
        'p': 'patient',
        'r': 'run',
        's': 'generic'
    }
24
    return mapping[target.split(':')[1][0]]
25

Ryan Herbert's avatar
Ryan Herbert committed
26

27
def manage_filename(filename):
28 29
    filepath = ""
    name_list = []
Ryan Herbert's avatar
Ryan Herbert committed
30 31 32
    name_list = filename.split('/')
    myfilename = name_list[-1]
    data = dict(filename=myfilename, data_file=None)
33 34

    if len(name_list) > 1:
Ryan Herbert's avatar
Ryan Herbert committed
35 36
        filepath = defs.FILE_SOURCE + '/' + filename
        split_file = myfilename.split('.')
37 38
        uuid_key = db.uuid().replace('-', '')[-16:]
        encoded_filename = base64.b16encode('.'.join(split_file[0:-1])).lower()
39
        data_file = "sequence_file.data_file.%s.%s.%s" % (
40 41
                uuid_key, encoded_filename, split_file[-1]
            )
42 43 44
        data['data_file'] = data_file

    return (data, filepath)
45

46 47 48 49 50
def link_to_sample_sets(seq_file_id, id_dict):
    '''
    Create sample set memberships and return a dict of the sample set ids.
    The keys to the dict are thee same as the ones passed in id_dict
    '''
51
    log.debug("linking file %d to sets:" % seq_file_id)
52
    for key in id_dict:
53
        log.debug("%s: %s" % (key, str(id_dict[key])))
54 55
        arr = [{'sample_set_id': oid, 'sequence_file_id': seq_file_id} for oid in id_dict[key]]
        db.sample_set_membership.bulk_insert(arr)
56
    db.commit()
57

58 59 60 61 62 63 64 65 66 67 68 69
# TODO put these in a model or utils or smth
def validate(myfile):
    error = []
    if myfile['filename'] == None :
        error.append("missing filename")
    if myfile['sampling_date'] != '' :
        try:
            datetime.datetime.strptime(""+myfile['sampling_date'], '%Y-%m-%d')
        except ValueError:
            error.append("date (wrong format)")
    return error

70 71 72 73
def validate_sets(set_ids):
    id_dict = {}
    sets = []
    errors = []
74

75
    if len(set_ids) == 0:
76 77 78 79 80
        errors.append("missing set association")

    mf = ModelFactory()
    helpers = {}

81 82
    set_ids_arr = []
    if len(set_ids) > 0:
83
        set_ids_arr = [x.strip() for x in set_ids.split('|')]
84
    for sid in set_ids_arr:
85 86
        try:
            set_type = extract_set_type(sid)
87
            if set_type not in id_dict:
88
                helpers[set_type] = mf.get_instance(set_type)
89 90
                id_dict[set_type] = []
            sets.append({'type': set_type, 'id': sid})
91
            set_id = helpers[set_type].parse_id_string(sid)
92
            id_dict[set_type].append(set_id)
93
            if not auth.can_modify_sample_set(set_id) :
94 95
                errors.append("missing permission for %s %d" % (set_type, set_id))
        except ValueError:
Ryan Herbert's avatar
Ryan Herbert committed
96
            errors.append("Invalid %s %s" % (set_type, sid))
97
    return sets, id_dict, errors
98 99

def get_pre_process_list():
100
    query_pre_process = db((auth.vidjil_accessible_query(PermissionEnum.read_pre_process.value, db.pre_process) | auth.vidjil_accessible_query(PermissionEnum.admin_pre_process.value, db.pre_process) ) ).select(orderby=~db.pre_process.id)
101 102 103 104 105 106 107 108 109 110 111 112 113

    pre_process_list = []
    for row in query_pre_process :
        file = 1
        if "&file2" in row.command:
                file = 2
        pre_process_list.append(dict(
                        id = row.id,
                        name = row.name,
                        file = file,
                        info = row.info
                ))
    return pre_process_list
114

115 116 117 118 119 120 121 122
def get_set_list(id_dict, helpers):
    sets = []
    for key in id_dict:
        slist = db(db[key].id.belongs(id_dict[key])).select()
        for sset in slist:
            sets.append({'type': key, 'id': helpers[key].get_id_string(sset)})
    return sets

123
def get_set_helpers():
Ryan Herbert's avatar
Ryan Herbert committed
124 125 126 127 128
    factory = ModelFactory()
    sample_types = [defs.SET_TYPE_GENERIC, defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN]
    helpers = {}
    for stype in sample_types:
        helpers[stype] = factory.get_instance(type=stype)
129 130 131 132 133 134 135
    return helpers

def form():
    group_ids = []
    relevant_ids = {}

    helpers = get_set_helpers()
Ryan Herbert's avatar
Ryan Herbert committed
136 137 138 139 140 141

    # new file
    if 'sample_set_id' in request.vars:
        sample_set = db.sample_set[request.vars["sample_set_id"]]
        if not auth.can_upload_sample_set(sample_set.id):
            return error_message("you don't have right to upload files")
142

Ryan Herbert's avatar
Ryan Herbert committed
143 144 145 146 147 148 149
        sample_type = sample_set.sample_type
        enough_space = vidjil_utils.check_enough_space(defs.DIR_SEQUENCES)
        if not enough_space:
            mail.send(to=defs.ADMIN_EMAILS,
                subject="[Vidjil] Server space",
                message="The space in directory %s has passed below %d%%." % (defs.DIR_SEQUENCES, defs.FS_LOCK_THRESHHOLD))
            return error_message("Uploads are temporarily disabled. System admins have been made aware of the situation.")
150

Ryan Herbert's avatar
Ryan Herbert committed
151
        row = db(db[sample_set.sample_type].sample_set_id == request.vars["sample_set_id"]).select().first()
Ryan Herbert's avatar
Ryan Herbert committed
152 153 154 155
        stype = sample_set.sample_type
        if stype not in relevant_ids:
            relevant_ids[stype] = []
        relevant_ids[stype].append(row.id)
156
        action = 'add'
157 158
        log.debug("load add form", extra={'user_id': auth.user.id,
                'record_id': request.vars['sample_set_id'],
159
                'table_name': "sample_set"})
160

Ryan Herbert's avatar
Ryan Herbert committed
161 162 163 164
    # edit file
    elif 'file_id' in request.vars:
        if not auth.can_modify_file(request.vars['file_id']):
            return error_message("you need admin permission to edit files")
HERBERT Ryan's avatar
HERBERT Ryan committed
165 166

        sample_set_list = db(
Ryan Herbert's avatar
Ryan Herbert committed
167
                (db.sample_set_membership.sequence_file_id == request.vars['file_id'])
HERBERT Ryan's avatar
HERBERT Ryan committed
168
                & (db.sample_set_membership.sample_set_id != None)
169 170
                & (db.sample_set.id == db.sample_set_membership.sample_set_id)
                & (db.sample_set.sample_type != 'sequence_file')
Ryan Herbert's avatar
Ryan Herbert committed
171
            ).select(db.sample_set_membership.sample_set_id.with_alias('sample_set_id'), db.sample_set.sample_type.with_alias('sample_type'))
172
        for row in sample_set_list :
Ryan Herbert's avatar
Ryan Herbert committed
173
            smp_type= row.sample_type
Ryan Herbert's avatar
Ryan Herbert committed
174 175 176
            if smp_type not in relevant_ids:
                relevant_ids[smp_type] = []
            relevant_ids[smp_type].append(db(db[smp_type].sample_set_id == row.sample_set_id).select()[0].id)
177
        action = 'edit'
178

Ryan Herbert's avatar
Ryan Herbert committed
179
        sample_type = request.vars["sample_type"]
180 181
        log.debug("load edit form", extra={'user_id': auth.user.id,
                'record_id': request.vars['file_id'],
182
                'table_name': "sequence_file"})
183
    else:
Ryan Herbert's avatar
Ryan Herbert committed
184 185
        return error_message("missing sample_set or file id")

186 187 188
    myfile = db.sequence_file[request.vars["file_id"]]
    if myfile is None:
        myfile = {}
189
    myfile['sets'] = []
190
    sets = get_set_list(relevant_ids, helpers)
Ryan Herbert's avatar
Ryan Herbert committed
191

192 193 194 195 196
    data = {}
    data['file'] = [myfile]
    data['sets'] = sets
    data['sample_type'] = sample_type
    data['errors'] = []
197
    data['action'] = action
Ryan Herbert's avatar
Ryan Herbert committed
198

199
    return form_response(data)
200 201

#TODO check data
Ryan Herbert's avatar
Ryan Herbert committed
202
def submit():
203 204
    data = json.loads(request.vars['data'], encoding='utf-8')
    error = False
205 206

    pre_process = None
207
    pre_process_flag = "COMPLETED"
208 209
    if 'pre_process' in data and data['pre_process'] is not None and\
       int(data['pre_process']) > 0:
210 211
        pre_process = int(data['pre_process'])
        pre_process_flag = "WAIT"
212

213
    sets, common_id_dict, errors = validate_sets(data['set_ids'])
214 215 216 217
    data['sets'] = sets

    data['errors'] = errors

218
    data['action'] = 'add'
219
    if len(errors) > 0:
220
        error = True
221

222
    for f in data['file']:
223
        f['errors'] = validate(f)
224

225 226
        f['sets'], f['id_dict'], err = validate_sets(f['set_ids'])

227
        if len(f['errors']) > 0:
228 229 230 231 232
            error = True
            continue

        file_data = dict(sampling_date=f['sampling_date'],
                         info=f['info'],
233
                         pre_process_id=pre_process,
234 235 236 237 238 239 240 241
                         pre_process_flag= pre_process_flag,
                         provider=auth.user_id)

        # edit
        if (f["id"] != ""):
            reupload = True
            fid = int(f["id"])
            sequence_file = db.sequence_file[fid]
242 243 244 245
            if f['filename'] == '':
                # If we don't reupload a new file
                file_data.pop('pre_process_flag')
            
246 247 248
            db.sequence_file[fid] = file_data
            #remove previous membership
            db( db.sample_set_membership.sequence_file_id == fid).delete()
249
            action = "edit"
250 251 252 253

        # add
        else:
            reupload = False
254
            f['id'] = fid = db.sequence_file.insert(**file_data)
255
            action = "add"
256

257
        data['action'] = action
258
        f['message'] = []
259
        mes = "file (%d) %s %sed" % (int(f["id"]), f["filename"], action)
Ryan Herbert's avatar
Ryan Herbert committed
260 261
        f['message'].append(mes)
        f['message'].append("You must reselect the file for it to be uploaded")
262

263 264
        id_dict = common_id_dict.copy()

265 266 267 268 269
        for key in f['id_dict']:
            if key not in id_dict:
                id_dict[key] = []
            id_dict[key] += f['id_dict'][key]

270 271
        for key in id_dict:
            for sid in id_dict[key]:
272 273
                group_id = get_set_group(sid)
                register_tags(db, 'sequence_file', fid, f["info"], group_id, reset=True)
274 275 276 277 278

        if f['filename'] != "":
            if reupload:
                # file is being reuploaded, remove all existing results_files
                db(db.results_file.sequence_file_id == fid).delete()
279
                mes += " file was replaced"
280 281 282 283 284 285 286

            file_data, filepath = manage_filename(f["filename"])
            filename = file_data['filename']
            if 'data_file' in file_data and file_data['data_file'] is not None:
                os.symlink(filepath, defs.DIR_SEQUENCES + file_data['data_file'])
                file_data['size_file'] = os.path.getsize(filepath)
                file_data['network'] = True
Ryan Herbert's avatar
Ryan Herbert committed
287
                file_data['data_file'] = str(file_data['data_file'])
288 289
            db.sequence_file[fid] = file_data

290
        link_to_sample_sets(fid, id_dict)
291

292 293
        log.info(mes, extra={'user_id': auth.user.id,
                'record_id': f['id'],
294
                'table_name': "sequence_file"})
Ryan Herbert's avatar
Ryan Herbert committed
295

296
    if not error:
297 298
        set_type = data['sets'][0]['type']
        set_id = id_dict[set_type][0]
299
        res = { "file_ids": [f['id'] for f in data['file']],
300 301
                "redirect": "sample_set/index",
                "args" : { "id" : set_id, "config_id": -1},
302 303
                "message": "successfully added/edited file(s)"}
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
304
    else:
305
        return form_response(data)
306
    
307 308
def form_response(data):
    source_module_active = hasattr(defs, 'FILE_SOURCE') and hasattr(defs, 'FILE_TYPES')
309 310 311 312
    network_source = source_module_active and (data['action'] != 'edit'     \
                                               or len(data['file']) == 0    \
                                               or data['file'][0].network)
      # should be true only when we want to use the network view
313 314
    response.view = 'file/form.html'
    upload_group_ids = [int(gid) for gid in get_upload_group_ids(auth)]
315
    group_ids = get_involved_groups()
316 317 318 319 320 321 322 323
    pre_process_list = get_pre_process_list()
    return dict(message=T("an error occured"),
           pre_process_list = pre_process_list,
           files = data['file'],
           sets = data['sets'],
           sample_type = data['sample_type'],
           errors = data['errors'],
           source_module_active = source_module_active,
324
           network_source = network_source,
325
           group_ids = group_ids,
326 327
           upload_group_ids = upload_group_ids,
           isEditing = data['action']=='edit')
328

329
def upload(): 
Marc Duez's avatar
Marc Duez committed
330
    session.forget(response)
331
    mes = ""
332
    error = ""
333

334 335
    if request.vars['id'] == None :
        error += "missing id"
336 337
    elif db.sequence_file[request.vars["id"]] is None:
        error += "no sequence file with this id"
338

339
    if not error:
340
        mes += " file {%s} " % (request.vars['id'])
341
        res = {"message": mes + "processing uploaded file"}
342
        log.debug(res)
343
        if request.vars.file != None :
Marc Duez's avatar
Marc Duez committed
344
            f = request.vars.file
345
            try:
346 347 348 349
                if request.vars["file_number"] == "1" :
                    db.sequence_file[request.vars["id"]] = dict(data_file = db.sequence_file.data_file.store(f.file, f.filename))
                else :
                    db.sequence_file[request.vars["id"]] = dict(data_file2 = db.sequence_file.data_file.store(f.file, f.filename))
350
                mes += "upload finished (%s)" % (f.filename)
351 352
            except IOError as e:
                if str(e).find("File name too long") > -1:
Mikaël Salson's avatar
Mikaël Salson committed
353
                    error += 'Your filename is too long, please shorten it.'
354
                else:
Mikaël Salson's avatar
Mikaël Salson committed
355
                    error += "System error during processing of uploaded file."
356
                    log.error(str(e))
357
        
358
        data_file = db.sequence_file[request.vars["id"]].data_file
359 360 361
        data_file2 = db.sequence_file[request.vars["id"]].data_file2
        
        if request.vars["file_number"] == "1" and len(error) == 0 and data_file is None:
362
            error += "no data file"
363 364
        if request.vars["file_number"] == "2" and len(error) == 0 and data_file2 is None:
            error += "no data file"
365 366 367

        db.sequence_file[request.vars["id"]] = dict(pre_process_flag=None,
                                                    pre_process_result=None)
368
        if data_file is not None and data_file2 is not None and request.vars['pre_process'] != '0':
369
            db.sequence_file[request.vars["id"]] = dict(pre_process_flag = "WAIT")
370 371 372 373 374
            old_task_id = db.sequence_file[request.vars["id"]].pre_process_scheduler_task_id
            if db.scheduler_task[old_task_id] != None:
                scheduler.stop_task(old_task_id)
                db(db.scheduler_task.id == old_task_id).delete()
                db.commit()
375
            schedule_pre_process(int(request.vars['id']), int(request.vars['pre_process']))
376
            mes += " | p%s start pre_process %s " % (request.vars['pre_process'], request.vars['id'] + "-" +request.vars['pre_process'])
377

378 379 380 381 382 383
        if data_file is not None :
            seq_file = defs.DIR_SEQUENCES + data_file
            # Compute and store file size
            size = os.path.getsize(seq_file)
            mes += ' (%s)' % vidjil_utils.format_size(size)
            db.sequence_file[request.vars["id"]] = dict(size_file = size)
384

385 386 387 388
        if data_file2 is not None :
            seq_file2 = defs.DIR_SEQUENCES + data_file2
            #TODO
        
389
    # Log and exit
Mikaël Salson's avatar
Mikaël Salson committed
390
    res = {"message": error + mes}
391
    if error:
Mikaël Salson's avatar
Mikaël Salson committed
392
        res['success'] = 'false'
393
        res['priority'] = 3
394 395
        log.error(res)
    else:
Mathieu Giraud's avatar
Mathieu Giraud committed
396
        log.info(res)
397
        log.debug("#TODO log all relevant info to database")
398
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Marc Duez's avatar
Marc Duez committed
399
  
400

401
def confirm():
402 403 404 405 406
    '''
    Request parameters:
    \param delete_results: (optional) boolean
    \param id: sequence file ID
    '''
407 408 409 410
    delete_only_sequence = ('delete_only_sequence' in request.vars and request.vars['delete_only_sequence'] == 'True')
    delete_results = ('delete_results' in request.vars and request.vars['delete_results'] == 'True')
    sequence_file = db.sequence_file[request.vars['id']]
    if sequence_file == None:
411
        return error_message("The requested file doesn't exist")
412 413
    if sequence_file.data_file == None:
        delete_results = True
414
    if auth.can_modify_sample_set(request.vars['redirect_sample_set_id']):
415 416 417
        return dict(message=T('choose what you would like to delete'),
                    delete_only_sequence = delete_only_sequence,
                    delete_results = delete_results)
418
    else:
419
        return error_message("you need admin permission to delete this file")
420

421 422 423
def delete_sequence_file(seq_id):
    sequence = db.sequence_file[seq_id]
    seq_filename = sequence.data_file
424 425

    if auth.can_modify_file(seq_id):
426 427
        if seq_filename is not None:
            log.debug('Deleting '+defs.DIR_SEQUENCES+seq_filename+' with ID'+str(seq_id))
428 429 430 431
        db.sequence_file[seq_id] = dict(data_file = None)
    else:
        return error_message('you need admin permission to delete this file')

432
def delete():
433 434 435 436 437
    '''
    Called (via request) with:
    \param: id (the sequence ID)
    \param: delete_results: (optional) boolean stating if we also want to delete the results.
    '''
438
    delete_results = ('delete_results' in request.vars and request.vars['delete_results'] == "True")
439 440
    sample_set = db.sample_set[request.vars["redirect_sample_set_id"]]
    associated_id = None
441
    if sample_set.sample_type not in ['sequence_file', 'sample_set']:
442
        associated_elements = db(db[sample_set.sample_type].sample_set_id == sample_set.id).select()
443
        if len(associated_elements) > 0:
444
            associated_id = associated_elements[0].id
445

446
    if auth.can_modify_file(request.vars["id"]):
447 448 449
        if not(delete_results):
            delete_sequence_file(request.vars['id'])
        else:
450 451
            sample_set_ids = get_sequence_file_sample_sets(request.vars["id"])
            config_ids = get_sequence_file_config_ids(request.vars["id"])
452
            db(db.results_file.sequence_file_id == request.vars["id"]).delete()
453
            db(db.sequence_file.id == request.vars["id"]).delete()
454
            schedule_fuse(sample_set_ids, config_ids)
455

456 457
        res = {"redirect": "sample_set/index",
               "args" : { "id" : request.vars["redirect_sample_set_id"]},
458
               "message": "sequence file ({}) deleted".format(request.vars['id'])}
459
        if associated_id is not None:
460
            log.info(res, extra={'user_id': auth.user.id, 'record_id': associated_id, 'table_name': sample_set.sample_type})
461 462
        else:
            log.info(res)
463 464
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    else:
465
        return error_message("you need admin permission to delete this file")
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492

def sequencer_list():
    sequencer_list = []
    for row in db(db.sequence_file.sequencer != None).select(db.sequence_file.sequencer, distinct=True):
        if row.sequencer is not "null" :
            sequencer_list.append(row.sequencer)
            
    res = {"sequencer": sequencer_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def pcr_list():
    pcr_list = []
    for row in db(db.sequence_file.pcr != None).select(db.sequence_file.pcr, distinct=True):
        if row.pcr is not "null" :
            pcr_list.append(row.pcr)
            
    res = {"pcr": pcr_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def producer_list():
    producer_list = []
    for row in db(db.sequence_file.producer != None).select(db.sequence_file.producer, distinct=True):
        if row.producer is not "null" :
            producer_list.append(row.producer)
            
    res = {"producer": producer_list}
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
493 494

def restart_pre_process():
495
    if "sequence_file_id" not in request.vars:
496 497 498 499
        return error_message("missing parameter")
    sequence_file = db.sequence_file[request.vars["sequence_file_id"]]
    if sequence_file is None or not auth.can_modify_file(sequence_file.id):
        return error_message("Permission denied")
500
    pre_process = db.pre_process[sequence_file.pre_process_id]
501 502
    db.sequence_file[sequence_file.id] = dict(pre_process_flag = 'WAIT')
    db.commit()
503
    res = schedule_pre_process(sequence_file.id, pre_process.id)
504
    log.debug("restart pre process", extra={'user_id': auth.user.id,
505
                'record_id': sequence_file.id,
506
                'table_name': "sequence_file"})
507
    return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
508

509 510
def match_filetype(filename, extension):
    ext_len = len(extension)
Mikaël Salson's avatar
Mikaël Salson committed
511
    return ext_len == 0 or filename[-ext_len:] == extension
512

Ryan Herbert's avatar
Ryan Herbert committed
513
def filesystem():
514
    json = []
515 516 517 518 519 520
    id = "" if request.vars["node"] is None else request.vars["node"] + '/'
    if id == "":
        json = [{"text": "/", "id": "/",  "children": True}]
    else:
        root_folder = defs.FILE_SOURCE + id
        for idx, f in enumerate(os.listdir(root_folder)):
Ryan Herbert's avatar
Ryan Herbert committed
521 522 523 524 525
            correct_type = False
            for ext in defs.FILE_TYPES:
                correct_type = match_filetype(f, ext)
                if correct_type:
                    break
526 527 528 529 530
            is_dir = os.path.isdir(root_folder + f)
            if correct_type or is_dir:
                json_node = jstree.Node(f, id + f).jsonData()
                if is_dir : json_node['children'] = True
                if correct_type: json_node['icon'] = 'jstree-file'
531
                json_node['li_attr']['title'] = f
532 533
                json.append(json_node)
    return gluon.contrib.simplejson.dumps(json, separators=(',',':'))