sample_set.py 37.3 KB
Newer Older
1 2 3 4
# coding: utf8
import gluon.contrib.simplejson, datetime
import vidjil_utils
import time
5
import json
6 7
from vidjilparser import VidjilParser
import operator
8
import math
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28

if request.env.http_origin:
    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin  
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400

ACCESS_DENIED = "access denied"




#
#
def next_sample_set():
    '''
    Process request, possibly changing request.vars['id'] depending on request.vars['next']
    '''
    if 'next' in request.vars:
        try:
            sample_type = db.sample_set[request.vars["id"]].sample_type
29
            sample_set_id = int(request.vars['id'])
30 31
            
            go_next = int(request.vars['next'])
32
            same_type_with_permissions = (db.sample_set.sample_type == sample_type) & (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set))
33
            if go_next > 0:
34
                res = db((db.sample_set.id > sample_set_id) & (same_type_with_permissions)).select(
35
                    db.sample_set.id, orderby=db.sample_set.id, limitby=(0,1))
36
            else:
37
                res = db((db.sample_set.id < sample_set_id) & (same_type_with_permissions)).select(
38
                    db.sample_set.id, orderby=~db.sample_set.id, limitby=(0,1))
39
            if (len(res) > 0):
40
                request.vars["id"] = str(res[0].id)
41 42 43 44 45 46 47 48
        except:
            pass

## return patient file list
##
def index():

    next_sample_set()
49 50 51 52 53
    if not auth.can_view_sample_set(request.vars["id"]):
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

54 55 56 57 58 59
    sample_set = db.sample_set[request.vars["id"]]
    sample_set_id = sample_set.id
    factory = ModelFactory()
    helper = factory.get_instance(type=sample_set.sample_type)
    data = helper.get_data(sample_set_id)
    info_file = helper.get_info_dict(data)
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77

    if request.vars["config_id"] and request.vars["config_id"] != "-1" and request.vars["config_id"] != "None":
        config_id = long(request.vars["config_id"])
        config_name = db.config[request.vars["config_id"]].name

        fused = db(
            (db.fused_file.sample_set_id == sample_set_id)
            & (db.fused_file.config_id == config_id)
        )

        analysis = db(
            db.analysis_file.sample_set_id == sample_set_id
        ).select(orderby=~db.analysis_file.analyze_date)
        
        
        config = True
        fused_count = fused.count()
        fused_file = fused.select()
78
        fused_filename = info_file["filename"] +"_"+ config_name + ".vidjil"
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
        analysis_count = len(analysis)
        analysis_file = analysis
        analysis_filename = info_file["filename"]+"_"+ config_name + ".analysis"
        
    else:
        config_id = -1
        config = False
        fused_count = 0
        fused_file = ""
        fused_filename = ""
        analysis_count = 0
        analysis_file = ""
        analysis_filename = ""

    if config :
	query =[]
	
        query2 = db(
                (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                & (db.sample_set_membership.sample_set_id == sample_set_id)
            ).select(
                left=db.results_file.on(
                    (db.results_file.sequence_file_id==db.sequence_file.id)
                    & (db.results_file.config_id==str(config_id) )
                ), 
                orderby = db.sequence_file.id|~db.results_file.run_date
            )
        
	previous=-1
	for row in query2 :
	    if row.sequence_file.id != previous : 
		query.append(row)
		previous=row.sequence_file.id

    else:

        query = db(
                (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                & (db.sample_set_membership.sample_set_id == sample_set_id)
            ).select(
                left=db.results_file.on(
                    (db.results_file.sequence_file_id==db.sequence_file.id)
                    & (db.results_file.config_id==str(config_id) )
                )
            )

125
    tag_decorator = TagDecorator(get_tag_prefix())
126 127 128 129
    query_pre_process = db( db.pre_process.id >0 ).select()
    pre_process_list = {}
    for row in query_pre_process:
        pre_process_list[row.id] = row.name
130 131 132 133
    
    log.debug('sample_set (%s)' % request.vars["id"])
    #if (auth.can_view_patient(request.vars["id"]) ):
    return dict(query=query,
134
                pre_process_list=pre_process_list,
135 136 137
                config_id=config_id,
                info=info_file,
                can_modify=auth.can_modify_sample_set(sample_set_id),
138
                can_upload=auth.can_upload_sample_set(sample_set_id),
139 140 141 142 143 144
                fused_count=fused_count,
                fused_file=fused_file,
                fused_filename=fused_filename,
                analysis_count=analysis_count,
                analysis_file = analysis_file,
                analysis_filename = analysis_filename,
145
                sample_type = db.sample_set[request.vars["id"]].sample_type,
146 147
                config=config,
                tag_decorator=tag_decorator)
148

149 150 151
## return a list of generic sample_sets
def all():
    start = time.time()
152 153 154 155 156
    if request.vars['type']:
        type = request.vars['type']
    else :
        type = defs.SET_TYPE_GENERIC

157 158
    if not auth.user :
        res = {"redirect" : URL('default', 'user', args='login', scheme=True, host=True,
159
                    vars=dict(_next=URL('sample_set', 'all', vars={'type': type, 'page': 0}, scheme=True, host=True)))
160 161 162 163
            }
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

    isAdmin = auth.is_admin()
164

165 166 167 168 169 170 171
    step = None
    page = None
    is_not_filtered = "sort" not in request.vars and "filter" not in request.vars
    if request.vars['page'] is not None and is_not_filtered:
        page = int(request.vars['page'])
        step = 50

172 173 174 175 176
    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
177
    group_ids = get_involved_groups()
178 179

    list = SampleSetList(type, page, step, tags=tags)
180 181 182
    list.load_creator_names()
    list.load_sample_information()
    list.load_config_information()
183
    if isAdmin or len(get_group_list(auth)) > 1:
184
        list.load_permitted_groups()
185 186
    list.load_anon_permissions()
    result = list.get_values()
187

188 189
    # failsafe if filtered display all results
    step = len(list) if step is None else step
190
    page = 0 if page is None else page
191

HERBERT Ryan's avatar
HERBERT Ryan committed
192 193 194
    factory = ModelFactory()
    helper = factory.get_instance(type=type)
    fields = helper.get_fields()
195
    sort_fields = helper.get_sort_fields()
196

197 198 199 200
    ##sort result
    reverse = False
    if request.vars["reverse"] == "true" :
        reverse = True
HERBERT Ryan's avatar
HERBERT Ryan committed
201
    if "sort" in request.vars:
202
        result = sorted(result, key = sort_fields[request.vars["sort"]]['call'], reverse=reverse)
203 204 205
    else:
        result = sorted(result, key = lambda row : row.id, reverse=not reverse)

206 207
    result = helper.filter(search, result)
    log.debug("%s list (%.3fs) %s" % (request.vars["type"], time.time()-start, search))
208 209 210


    return dict(query = result,
HERBERT Ryan's avatar
HERBERT Ryan committed
211
                fields = fields,
212
                helper = helper,
213
                group_ids = group_ids,
214
                isAdmin = isAdmin,
215
                reverse = reverse,
216 217
                step = step,
                page = page)
218

219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
def stats():
    start = time.time()
    if not auth.user :
        res = {"redirect" : URL('default', 'user', args='login', scheme=True, host=True,
                    vars=dict(_next=URL('sample_set', 'all', vars={'type': defs.SET_TYPE_PATIENT}, scheme=True, host=True)))
            }
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

    isAdmin = auth.is_admin()
    if request.vars['type']:
        type = request.vars['type']
    else :
        type = defs.SET_TYPE_GENERIC

    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
    group_ids = get_involved_groups()

    list = SampleSetList(type, tags=tags)
    list.load_sample_information()
    list.load_anon_permissions()
    result = list.get_values()

    factory = ModelFactory()
    helper = factory.get_instance(type=type)
    fields = helper.get_reduced_fields()

    ##sort result
    reverse = False
    if request.vars["reverse"] == "true" :
        reverse = True
    if "sort" in request.vars:
        result = sorted(result, key = lambda row : row[request.vars["sort"]], reverse=reverse)
    else:
        result = sorted(result, key = lambda row : row.id, reverse=not reverse)

    result = helper.filter(search, result)
    log.debug("%s stat list (%.3fs) %s" % (request.vars["type"], time.time()-start, search))

    return dict(query = result,
                fields = fields,
                helper = helper,
                group_ids = group_ids,
                isAdmin = isAdmin,
                reverse = False)

def result_files():
    from zipfile import ZipFile
    from cStringIO import StringIO
    import types
    errors = []
    config_id = request.vars['config_id']
    sample_set_ids = []
    if 'sample_set_ids' in request.vars:
        sample_set_ids = request.vars['sample_set_ids']

    #little hack since we can't pass array parameters with only one value
    if isinstance(sample_set_ids, types.StringTypes):
        sample_set_ids = [sample_set_ids]

282 283 284 285 286 287
    if int(config_id) == -1:
        config_query = (db.results_file.config_id > 0)
    else:
        config_query = (db.results_file.config_id == config_id)


288 289 290 291 292 293 294 295
    left_join = [
        db.patient.on(db.patient.sample_set_id == db.sample_set.id),
        db.run.on(db.run.sample_set_id == db.sample_set.id),
        db.generic.on(db.generic.sample_set_id == db.sample_set.id)
    ]
    q = db(
            (db.sample_set.id.belongs(sample_set_ids)) &
            (db.sample_set_membership.sample_set_id == db.sample_set.id) &
296 297
            (db.sequence_file.id == db.sample_set_membership.sequence_file_id) &
            (db.results_file.sequence_file_id == db.sequence_file.id) &
298
            (db.results_file.data_file != None) &
299
            config_query
300 301
        )

302
    results = q.select(db.results_file.ALL, db.sequence_file.ALL, db.sample_set.ALL, db.patient.ALL, db.run.ALL, db.generic.ALL, left=left_join)
303 304 305 306 307 308 309 310 311

    sample_types = ['patient', 'run', 'generic']
    mf = ModelFactory()
    helpers = {}

    for t in sample_types:
        helpers[t] = mf.get_instance(type=t)

    filename = "export_%s_%s.zip" % ('-'.join(sample_set_ids), str(datetime.date.today()))
312 313 314 315 316 317 318 319 320
    filedir = defs.DIR_SEQUENCES + '/' + filename
    try:
        zipfile = ZipFile(filedir, 'w')
        metadata = []
        for res in results:
            metadata.append({'id': res.sample_set.id,
                'name': helpers[res.sample_set.sample_type].get_name(res[res.sample_set.sample_type]),
                'file': res.results_file.data_file,
                'set_info': res[res.sample_set.sample_type].info,
321 322
                'sample_info': res.sequence_file.info,
                'sequence_file': res.sequence_file.filename})
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
            path = defs.DIR_RESULTS + res.results_file.data_file
            zipfile.writestr(res.results_file.data_file, open(path, 'rb').read())

        zipfile.writestr('metadata.json', json.dumps(metadata))

        zipfile.close()

        response.headers['Content-Type'] = "application/zip"
        response.headers['Content-Disposition'] = 'attachment; filename=%s' % filename# to force download as attachment

        return response.stream(open(filedir), chunk_size=4096)
    except:
        res = {"message": "an error occurred"}
        log.error("An error occured when creating archive of sample_sets %s" % str(sample_set_ids))
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    finally:
        try:
            os.unlink(filedir)
        except OSError:
            pass
343

344

345 346
## Stats

347
def mystats():
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
    start = time.time()

    # d = all()
    d = custom()
    
    # Build .vidjil file list
    f_samples = []
    for row in d['query']:
        found = {}
        f_results = defs.DIR_RESULTS + row.results_file.data_file

        f_fused = None
        pos_in_fused = None

        fused_file = ''
        # TODO: fix the following request
        # fused_file = db((db.fused_file.sample_set_id == row.sample_set.id) & (db.fused_file.config_id == row.results_file.config_id)).select(orderby = ~db.fused_file.id, limitby=(0,1))
        if len(fused_file) > 0 and fused_file[0].sequence_file_list is not None:
            sequence_file_list = fused_file[0].sequence_file_list.split('_')
            try:
                pos_in_fused = sequence_file_list.index(str(row.sequence_file.id))
                f_fused = defs.DIR_RESULTS + fused_file[0].fused_file
            except ValueError:
                pass
                
        metadata = { } # 'patient': row.patient, 'sequence_file': row.sequence_file }
        f_samples += [(metadata, f_results, f_fused, pos_in_fused)]
    
    # Send to vidjil_utils.stats
    res = vidjil_utils.stats(f_samples)
    d = {}
    d['stats'] = res
    d['f_samples'] = f_samples # TMP, for debug

    # Return
    log.debug("stats (%.3fs) %s" % (time.time()-start, request.vars["filter"]))
    return gluon.contrib.simplejson.dumps(d, separators=(',',':'))

Ryan Herbert's avatar
Ryan Herbert committed
386 387 388 389 390 391
## return form to create new set
def form():
    denied = False
    # edit set
    if("id" in request.vars):
        sample_set = db.sample_set[request.vars["id"]]
392 393
        set_type = sample_set.sample_type
        sset = db(db[set_type].sample_set_id == sample_set.id).select().first()
394 395
        if(auth.can_modify_sample_set(sset.sample_set_id)):
            groups = [get_set_group(sset.sample_set_id)]
396
            action = 'edit'
Ryan Herbert's avatar
Ryan Herbert committed
397 398 399 400 401 402 403 404 405
            max_group = None
        else:
            denied = True

    # new set
    elif (auth.can_create_patient()):
        sset = None
        set_type = request.vars["type"]
        creation_group_tuple = get_default_creation_group(auth)
406 407
        groups = creation_group_tuple[0]
        max_group = creation_group_tuple[1]
408
        action = 'add'
409
    else :
Ryan Herbert's avatar
Ryan Herbert committed
410 411 412
        denied = True

    if denied:
413 414 415 416
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

417
    message = '%s %s' % (action, set_type)
Ryan Herbert's avatar
Ryan Herbert committed
418 419 420 421 422 423 424 425 426
    sets = {
            'patient': [],
            'run': [],
            'generic': []
            }
    # We add a None object to the desired set type to initialise an empty form in the template.
    sets[set_type].append(sset)
    return dict(message=T(message),
                groups=groups,
427
                group_ids = get_involved_groups(),
Ryan Herbert's avatar
Ryan Herbert committed
428
                master_group=max_group,
429 430
                sets=sets,
                isEditing = (action=='edit'))
Ryan Herbert's avatar
Ryan Herbert committed
431

432 433


Ryan Herbert's avatar
Ryan Herbert committed
434 435 436
## create a patient if the html form is complete
## need ["first_name", "last_name", "birth_date", "info"]
## redirect to patient list if success
437
## return a flash error message if fail
Ryan Herbert's avatar
Ryan Herbert committed
438 439 440
def submit():
    data = json.loads(request.vars['data'], encoding='utf-8')
    mf = ModelFactory()
441

Ryan Herbert's avatar
Ryan Herbert committed
442 443
    error = False
    set_types = vidjil_utils.get_found_types(data)
444 445

    length_mapping = {}
446
    sum_sets = 0
Ryan Herbert's avatar
Ryan Herbert committed
447 448
    for set_type in set_types:
        helper = mf.get_instance(set_type)
449
        length = len(data[set_type])
450
        sum_sets += length
451 452
        if length not in length_mapping:
            length_mapping[length] = set_type
Ryan Herbert's avatar
Ryan Herbert committed
453
        for p in data[set_type]:
454
            errors = helper.validate(p)
455
            action = "add"
456 457
            if len(errors) > 0:
                p['error'] = errors
Ryan Herbert's avatar
Ryan Herbert committed
458 459
                error = True
                continue
460

Ryan Herbert's avatar
Ryan Herbert committed
461 462
            register = False
            reset = False
HERBERT Ryan's avatar
HERBERT Ryan committed
463

Ryan Herbert's avatar
Ryan Herbert committed
464
            name = helper.get_name(p)
465

Ryan Herbert's avatar
Ryan Herbert committed
466
            # edit
467
            if (p['sample_set_id'] != "" and auth.can_modify_sample_set(p['sample_set_id'])):
Ryan Herbert's avatar
Ryan Herbert committed
468
                reset = True
469 470
                sset = db(db[set_type].sample_set_id == p['sample_set_id']).select().first()
                db[set_type][sset.id] = p
471
                id_sample_set = sset['sample_set_id']
472

Ryan Herbert's avatar
Ryan Herbert committed
473
                if (sset.info != p['info']):
474
                    group_id = get_set_group(id_sample_set)
Ryan Herbert's avatar
Ryan Herbert committed
475 476
                    register = True
                    reset = True
477

478
                action = "edit"
479

Ryan Herbert's avatar
Ryan Herbert committed
480 481
            # add
            elif (auth.can_create_patient()):
482

Ryan Herbert's avatar
Ryan Herbert committed
483
                id_sample_set = db.sample_set.insert(sample_type=set_type)
484

Ryan Herbert's avatar
Ryan Herbert committed
485 486 487
                p['creator'] = auth.user_id
                p['sample_set_id'] = id_sample_set
                p['id'] = db[set_type].insert(**p)
488

Ryan Herbert's avatar
Ryan Herbert committed
489
                group_id = int(data["group"])
490

Ryan Herbert's avatar
Ryan Herbert committed
491
                register = True
492

Ryan Herbert's avatar
Ryan Herbert committed
493
                #patient creator automaticaly has all rights
494
                auth.add_permission(group_id, PermissionEnum.access.value, 'sample_set', p['sample_set_id'])
Ryan Herbert's avatar
Ryan Herbert committed
495

496
                action = "add"
Ryan Herbert's avatar
Ryan Herbert committed
497 498 499 500 501 502 503 504 505 506

                if (p['id'] % 100) == 0:
                    mail.send(to=defs.ADMIN_EMAILS,
                    subject="[Vidjil] %d" % p['id'],
                    message="The %dth %s has just been created." % (p['id'], set_type))

            else :
                p['error'].append("permission denied")
                error = True

507
            p['message'] = []
508
            mes = u"%s (%s) %s %sed" % (set_type, id_sample_set, name, action)
Ryan Herbert's avatar
Ryan Herbert committed
509
            p['message'].append(mes)
510
            log.info(mes, extra={'user_id': auth.user.id, 'record_id': p['id'], 'table_name': 'patient'})
Ryan Herbert's avatar
Ryan Herbert committed
511 512 513 514
            if register:
                register_tags(db, set_type, p["id"], p["info"], group_id, reset=reset)

    if not error:
515 516 517 518
        max_num = max(length_mapping.keys())
        msg = "successfully added/edited set(s)"
        if sum_sets == 1:
            res = {"redirect": "sample_set/index",
519
                   "args": { "id":  data[length_mapping[max_num]][0]['sample_set_id']},
520 521 522
                   "message": msg}
        else:
            res = {"redirect": "sample_set/all",
523
                    "args" : { "type" : length_mapping[max_num], "page": 0 },
524
                    "message": msg}
525
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
526 527 528 529 530 531
    else:
        sets = {
                'patient': data['patient'] if 'patient' in data else [],
                'run': data['run'] if 'run' in data else [],
                'generic': data['generic'] if 'generic' in data else []
                }
532
        response.view = 'sample_set/form.html'
Ryan Herbert's avatar
Ryan Herbert committed
533 534 535
        return dict(message=T("an error occured"),
                groups=[{'name': 'foobar', 'id': int(data['group'])}],
                master_group=data['group'],
536 537
                sets=sets,
                isEditing = (action=='edit'))
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558

def custom():
    start = time.time()

    if request.vars["config_id"] and request.vars["config_id"] != "-1" :
        config_id = long(request.vars["config_id"])
        config_name = db.config[request.vars["config_id"]].name
        config = True
        
    else:
        request.vars["config_id"] = -1
        config_id = -1
        config_name = None
        config = False
        
    if "custom_list" not in request.vars :
        request.vars["custom_list"] = []
    if type(request.vars["custom_list"]) is str :
        request.vars["custom_list"] = [request.vars["custom_list"]]
        
    myGroupBy = None
559
    helper = None
560
    if request.vars["id"] and auth.can_view_sample_set(request.vars["id"]):
561 562 563
        sample_set = db.sample_set[request.vars["id"]]
        factory = ModelFactory()
        helper = factory.get_instance(type=sample_set.sample_type)
564
        qq = (db.sample_set.id == request.vars["id"])
565 566
        
    else:
567
        qq = (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set))
568
        myGroupBy = db.sequence_file.id|db.patient.id|db.run.id|db.generic.id|db.results_file.config_id
569

570 571 572 573 574 575 576 577
    q = (qq
        & (auth.vidjil_accessible_query(PermissionEnum.read_config.value, db.config))
        & (db.sample_set_membership.sample_set_id == db.sample_set.id)
        & (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
        & (db.results_file.sequence_file_id==db.sequence_file.id)
        & (db.results_file.data_file != '')
        & (db.config.id==db.results_file.config_id))

578 579
    group_ids = get_involved_groups()

580 581 582 583 584
    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
585

586 587 588 589 590 591 592
    left_join = [
        db.patient.on(db.patient.sample_set_id == db.sample_set.id),
        db.run.on(db.run.sample_set_id == db.sample_set.id),
        db.generic.on(db.generic.sample_set_id == db.sample_set.id)
    ]

    select = [
593
        db.patient.id, db.patient.sample_set_id, db.patient.info, db.patient.first_name, db.patient.last_name,
594 595 596 597 598 599 600 601
        db.run.id, db.run.info, db.run.name,
        db.generic.id, db.generic.info, db.generic.name,
        db.results_file.id, db.results_file.config_id, db.sequence_file.sampling_date,
        db.sequence_file.pcr, db.config.name, db.results_file.run_date, db.results_file.data_file, db.sequence_file.filename,
        db.sequence_file.data_file, db.sequence_file.id, db.sequence_file.info,
        db.sequence_file.size_file
    ]

602 603 604
    if (tags is not None and len(tags) > 0):
        q = filter_by_tags(q, 'sequence_file', tags)
        count = db.tag.name.count()
605
        select = select + [db.tag_ref.record_id,
606
                db.tag_ref.table_name,
607 608 609 610 611
                count]

        query = db(q).select(
                *select,
                left = left_join,
612 613 614 615 616 617 618
                orderby = db.sequence_file.id|db.results_file.run_date,
                groupby = db.tag_ref.table_name|db.tag_ref.record_id,
                having = count >= len(tags)
            )

    else:
        query = db(q).select(
619 620
                *select,
                left = left_join,
621
                orderby = db.sequence_file.id|db.results_file.run_date,
622 623 624 625 626 627 628 629
                groupby = myGroupBy
            )

    for row in query :
        row.checked = False
        if (str(row.results_file.id) in request.vars["custom_list"]) :
            row.checked = True

630
        if row.patient.id is not None:
Ryan Herbert's avatar
Ryan Herbert committed
631
            #TODO use helper.
632
            row.names = vidjil_utils.display_names(row.patient.sample_set_id, row.patient.first_name, row.patient.last_name)
633
            info = row.patient.info
634
        elif row.run.id is not None:
635 636
            row.names = row.run.name
            info = row.run.info
637 638 639
        elif row.generic.id is not None:
            row.names = row.generic.name
            info = row.generic.info
640
        row.string = [row.names, row.sequence_file.filename, str(row.sequence_file.sampling_date), str(row.sequence_file.pcr), str(row.config.name), str(row.results_file.run_date), info]
641
    query = query.find(lambda row : ( vidjil_utils.advanced_filter(row.string,search) or row.checked) )
642 643 644 645 646

    
    if config :
        query = query.find(lambda row : ( row.results_file.config_id==config_id or (str(row.results_file.id) in request.vars["custom_list"])) )
    
647
    tag_decorator = TagDecorator(get_tag_prefix())
648
    log.debug("sample_set/custom (%.3fs) %s" % (time.time()-start, search))
649 650 651

    return dict(query=query,
                config_id=config_id,
652
                config=config,
653
                helper=helper,
654 655
                tag_decorator=tag_decorator,
                group_ids=group_ids)
656

657
def getStatHeaders():
658
    m = StatDecorator()
659
    s = SetsDecorator()
660 661
    b = BooleanDecorator()
    p = BarDecorator()
662
    bc = BarChartDecorator()
663
    lbc = LabeledBarChartDecorator()
664
    g = GenescanDecorator()
665
    l = LociListDecorator()
666
    return [('sample_sets', 'db', s),
Vidjil's avatar
Vidjil committed
667 668
            #('reads', 'parser', m),
            ('mapped reads', 'parser', m),
Vidjil's avatar
Vidjil committed
669
            #('mapped_percent', 'parser', p),
670
            ('read lengths', 'parser', g),
671 672
            #('bool', 'parser', b),
            #('bool_true', 'parser', b),
Vidjil's avatar
Vidjil committed
673
            ('loci', 'parser', l),
674 675
            #('distribution', 'parser', lbc),
            #('clones_five_percent', 'parser', m),
Vidjil's avatar
Vidjil committed
676
            ('main clone', 'parser', m)
677
            #('abundance', 'parser', lbc)
678
        ]
679

680
def getFusedStats(file_name, res, dest):
681
    log.debug("getFusedStats()")
682
    file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
683
    log.debug("file_path: %s" % file_path)
684
    parser = VidjilParser()
685
    parser.addPrefix('clones.item', 'clones.item.top', operator.le, 100)
686 687
    parser.addPrefix("reads")
    parser.addPrefix("samples")
688 689

    mjson = parser.extract(file_path)
690 691 692 693 694 695
    data = json.loads(mjson)
    result_index = -1
    if "results_file_id" in data['samples']:
        result_index = data['samples']['results_file_id'].index(res['resuts_file_id'])
    elif "original_names" in data['samples']:
        result_index = data['samples']['original_names'].index(defs.DIR_SEQUENCES + res['sequence_file'])
Vidjil's avatar
Vidjil committed
696
    dest['main clone'] = data['clones'][0]['name']
697
    reads = data['reads']['total'][result_index]
Vidjil's avatar
Vidjil committed
698 699 700
    # dest['reads'] = reads
    mapped_reads = data['reads']['segmented'][result_index]
    dest['mapped reads'] = "%d / %d (%.2f %%)" % (mapped_reads, reads, 100.0*mapped_reads/reads if reads else 0)
701
    dest['mapped_percent'] = 100.0 * (float(data['reads']['segmented'][result_index])/float(reads))
702
    dest['abundance'] = [(key, 100.0*data['reads']['germline'][key][result_index]/reads) for key in data['reads']['germline']]
703 704 705 706 707 708

    tmp = {}
    for c in data['clones']:
        arl = int(math.ceil(c['_average_read_length'][result_index]))
        if arl > 0:
            if arl not in tmp:
709 710 711 712
                tmp[arl] = 0.0
            tmp[arl] += float(c['reads'][result_index])
    min_len = 0 #int(min(tmp.keys()))
    max_len = 500 #int(max(tmp.keys()))
713
    tmp_list = []
714 715 716

    if mapped_reads == 0:
        mapped_reads = 1
717 718
    for i in range(min_len, max_len):
        if i in tmp:
719
            if tmp[i]:
Vidjil's avatar
Vidjil committed
720
                scaled_val = (2.5 + math.log10(tmp[i]/mapped_reads)) / 2
721 722 723 724
                display_val = max(0.01, min(1, scaled_val)) * 100
            else:
                display_val = 0
            real_val = 100.0*(tmp[i]/mapped_reads)
725
        else:
726 727 728 729
            display_val = 0
            real_val = 0
        tmp_list.append((i, display_val, real_val))
    dest['read lengths'] = tmp_list
730

731 732
    #dest['bool'] = False
    #dest['bool_true'] = True
733
    dest['loci'] = sorted([x for x in data['reads']['germline'] if data['reads']['germline'][x][result_index] > 0])
734
    dest['clones_five_percent'] = sum([data['reads']['distribution'][key][result_index] for key in data['reads']['distribution']])
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
    return dest

def getResultsStats(file_name, dest):
    import ijson.backends.yajl2_cffi as ijson
    log.debug("getResultsStats()")
    file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
    log.debug("file_path: %s" % file_path)
    distributions = []
    with open(file_path, 'rb') as results:
        i = "1"
        while True:
            results.seek(0, 0)
            tmp =  [d for d in ijson.items(results, "reads-distribution-%s.item" % i)]
            if len(tmp) == 0:
                break
            else:
                distributions.append((i, tmp[0]))
                i += "0"
    dest['distribution'] = distributions
754 755
    return dest

756
def getStatData(results_file_ids):
757
    log.debug("getStatData(%s)" % str(results_file_ids))
758 759 760 761 762 763 764 765
    mf = ModelFactory()
    set_types = [defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN, defs.SET_TYPE_GENERIC]
    helpers = {}
    for stype in set_types:
        helpers[stype] = mf.get_instance(stype)

    query = db(
        (db.results_file.id.belongs(results_file_ids)) &
766
        (db.sequence_file.id == db.results_file.sequence_file_id) &
767 768 769 770 771
        (db.config.id == db.results_file.config_id) &
        (db.sample_set_membership.sequence_file_id == db.sequence_file.id) &
        (db.sample_set.id == db.sample_set_membership.sample_set_id) &
        (db.fused_file.sample_set_id == db.sample_set.id) &
        (db.fused_file.config_id == db.config.id)
772
        ).select(
773 774
            db.results_file.sequence_file_id, db.results_file.config_id,
            db.results_file.data_file.with_alias("data_file"), db.results_file.id.with_alias("results_file_id"),
775
            db.sequence_file.data_file.with_alias("sequence_file"),
776 777
            db.sample_set.id.with_alias("set_id"),
            db.sample_set.sample_type.with_alias("sample_type"),
778
            db.fused_file.fused_file.with_alias("fused_file"),
779
            db.patient.first_name, db.patient.last_name, db.patient.info.with_alias('set_info'), db.patient.sample_set_id,
780 781 782
            db.run.name,
            db.generic.name,
            db.config.name,
783 784

            db.generic.name.with_alias("set_name"), # use generic name as failsafe for set name
785 786 787 788 789 790 791
            left = [
                db.patient.on(db.patient.sample_set_id == db.sample_set.id),
                db.run.on(db.run.sample_set_id == db.sample_set.id),
                db.generic.on(db.generic.sample_set_id == db.sample_set.id)
            ]
        )

792
    tmp_data = {}
793
    for res in query:
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
        set_type = res['sample_type']
        if res.results_file_id not in tmp_data:
            tmp = res.copy()
            tmp['sample_sets'] = []
            tmp_data[tmp['results_file_id']] = tmp
            tmp.pop('set_id', None)
            tmp.pop(set_type, None)
            tmp.pop('set_info', None)
        else:
            tmp = tmp_data[res['results_file_id']]

        sample_set = {}
        sample_set['id'] = res['set_id']
        sample_set['name'] = helpers[set_type].get_name(res[set_type])
        sample_set['info'] = res['set_info']
809
        sample_set['type'] = set_type
810 811 812 813 814 815
        tmp['sample_sets'].append(sample_set)


    data = []
    for key in tmp_data:
        res = tmp_data[key]
816
        d = {}
817
        set_type = res['sample_type']
818
        headers = getStatHeaders()
819
        d = getFusedStats(res['fused_file'], res, d)
820
        #d = getResultsStats(res['data_file'], d)
821
        for head, htype, model in headers:
822
            if htype == 'db':
823 824
                d[head] = res[head]
            d[head] = model.decorate(d[head])
825 826
        d['sequence_file_id'] = res['results_file']['sequence_file_id']
        d['config_id'] = res['results_file']['config_id']
827 828 829 830 831
        data.append(d)
    return data

def multi_sample_stats():
    data = {}
832
    data['headers'] = [h for h, t, m in getStatHeaders()]
833
    results = []
834 835 836 837
    custom_result = request.vars['custom_result']
    if not isinstance(custom_result, list):
        custom_result = [custom_result]

838 839 840 841 842 843 844 845 846 847 848 849
    custom_result = [long(i) for i in custom_result]

    permitted_results = db(
        (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set)) &
        (db.sample_set.id == db.sample_set_membership.sample_set_id) &
        (db.sample_set_membership.sequence_file_id == db.results_file.sequence_file_id) &
        (db.results_file.id.belongs(custom_result))
    ).select(
            db.results_file.id.with_alias('results_file_id')
        )

    permitted_results_ids = [r.results_file_id for r in permitted_results]
850 851
    log.debug("multi_sample_stats premitted: " + str(permitted_results_ids))
    log.debug("multi_sample_stats requested: " + str(custom_result))
852 853 854 855 856
    if set(permitted_results_ids) != set(custom_result):
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

857
    results = getStatData(custom_result)
858 859 860
    data['results'] = results
    return dict(data=data)

861 862
def confirm():
    if auth.can_modify_sample_set(request.vars["id"]):
863 864 865 866
        sample_set = db.sample_set[request.vars["id"]]
        data = db(db[sample_set.sample_type].sample_set_id == sample_set.id).select().first()
        factory = ModelFactory()
        helper = factory.get_instance(type=sample_set.sample_type)
867 868
        log.debug('request sample_set deletion')
        return dict(message=T('confirm sample_set deletion'),
869 870
                    data=data,
                    helper=helper)
871 872 873 874 875 876 877
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def delete():
    if (auth.can_modify_sample_set(request.vars["id"]) ):
878
        sample_set = db.sample_set[request.vars["id"]]
879
        sample_type = sample_set.sample_type
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
        if sample_set is None:
            res = {"message": 'An error occured. This sample_set may have already been deleted'}
            log.error(res)
            return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

        #delete data file
        query = db( (db.sample_set_membership.sample_set_id == sample_set.id)
                    & (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                ).select(db.sequence_file.id)
        for row in query :
            db(db.results_file.sequence_file_id == row.id).delete()

        #delete sequence file
        query = db((db.sequence_file.id == db.sample_set_membership.sequence_file_id)
            & (db.sample_set_membership.sample_set_id == sample_set.id)
            ).select(db.sequence_file.id)
        for row in query :
            db(db.sequence_file.id == row.id).delete()

        #delete patient sample_set
        db(db.sample_set.id == sample_set.id).delete()

        res = {"redirect": "sample_set/all",
903
               "args": {"type": sample_type, "page": 0},
904 905 906 907 908 909 910 911
               "success": "true",
               "message": "sample set ("+str(request.vars["id"])+") deleted"}
        log.info(res, extra={'user_id': auth.user.id, 'record_id': request.vars["id"], 'table_name': 'sample_set'})
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982

#
def permission():
    if (auth.can_modify_sample_set(request.vars["id"]) ):
        sample_set = db.sample_set[request.vars["id"]]
        stype = sample_set.sample_type
        factory = ModelFactory()
        helper = factory.get_instance(type=stype)

        data = db(db[stype].sample_set_id == sample_set.id).select().first()

        query = db( db.auth_group.role != 'admin' ).select()

        for row in query :
            row.owner = row.role
            if row.owner[:5] == "user_" :
                id = int(row.owner[5:])
                row.owner = db.auth_user[id].first_name + " " + db.auth_user[id].last_name

            permissions = db(
                    (db.auth_permission.group_id == row.id) &
                    (db.auth_permission.record_id == 0) &
                    (db.auth_permission.table_name == db.sample_set)).select()
            row.perms = ', '.join(map(lambda x: x.name, permissions))

            row.parent_access = ', '.join(str(value) for value in auth.get_access_groups(db[stype], request.vars['id'], group=row.id))
            row.read =  auth.get_group_access(sample_set.sample_type,