sample_set.py 34.3 KB
Newer Older
1
2
3
4
# coding: utf8
import gluon.contrib.simplejson, datetime
import vidjil_utils
import time
5
import json
6
7
from vidjilparser import VidjilParser
import operator
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

if request.env.http_origin:
    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin  
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400

ACCESS_DENIED = "access denied"




#
#
def next_sample_set():
    '''
    Process request, possibly changing request.vars['id'] depending on request.vars['next']
    '''
    if 'next' in request.vars:
        try:
            sample_type = db.sample_set[request.vars["id"]].sample_type
28
            sample_set_id = int(request.vars['id'])
29
30
            
            go_next = int(request.vars['next'])
31
            same_type_with_permissions = (db.sample_set.sample_type == sample_type) & (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set))
32
            if go_next > 0:
33
                res = db((db.sample_set.id > sample_set_id) & (same_type_with_permissions)).select(
34
                    db.sample_set.id, orderby=db.sample_set.id, limitby=(0,1))
35
            else:
36
                res = db((db.sample_set.id < sample_set_id) & (same_type_with_permissions)).select(
37
                    db.sample_set.id, orderby=~db.sample_set.id, limitby=(0,1))
38
            if (len(res) > 0):
39
                request.vars["id"] = str(res[0].id)
40
41
42
43
44
45
46
47
        except:
            pass

## return patient file list
##
def index():

    next_sample_set()
48
49
50
51
52
    if not auth.can_view_sample_set(request.vars["id"]):
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

53
54
55
56
57
58
    sample_set = db.sample_set[request.vars["id"]]
    sample_set_id = sample_set.id
    factory = ModelFactory()
    helper = factory.get_instance(type=sample_set.sample_type)
    data = helper.get_data(sample_set_id)
    info_file = helper.get_info_dict(data)
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

    if request.vars["config_id"] and request.vars["config_id"] != "-1" and request.vars["config_id"] != "None":
        config_id = long(request.vars["config_id"])
        config_name = db.config[request.vars["config_id"]].name

        fused = db(
            (db.fused_file.sample_set_id == sample_set_id)
            & (db.fused_file.config_id == config_id)
        )

        analysis = db(
            db.analysis_file.sample_set_id == sample_set_id
        ).select(orderby=~db.analysis_file.analyze_date)
        
        
        config = True
        fused_count = fused.count()
        fused_file = fused.select()
77
        fused_filename = info_file["filename"] +"_"+ config_name + ".vidjil"
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        analysis_count = len(analysis)
        analysis_file = analysis
        analysis_filename = info_file["filename"]+"_"+ config_name + ".analysis"
        
    else:
        config_id = -1
        config = False
        fused_count = 0
        fused_file = ""
        fused_filename = ""
        analysis_count = 0
        analysis_file = ""
        analysis_filename = ""

    if config :
	query =[]
	
        query2 = db(
                (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                & (db.sample_set_membership.sample_set_id == sample_set_id)
            ).select(
                left=db.results_file.on(
                    (db.results_file.sequence_file_id==db.sequence_file.id)
                    & (db.results_file.config_id==str(config_id) )
                ), 
                orderby = db.sequence_file.id|~db.results_file.run_date
            )
        
	previous=-1
	for row in query2 :
	    if row.sequence_file.id != previous : 
		query.append(row)
		previous=row.sequence_file.id

    else:

        query = db(
                (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                & (db.sample_set_membership.sample_set_id == sample_set_id)
            ).select(
                left=db.results_file.on(
                    (db.results_file.sequence_file_id==db.sequence_file.id)
                    & (db.results_file.config_id==str(config_id) )
                )
            )

124
    tag_decorator = TagDecorator(get_tag_prefix())
125
126
127
128
    query_pre_process = db( db.pre_process.id >0 ).select()
    pre_process_list = {}
    for row in query_pre_process:
        pre_process_list[row.id] = row.name
129
130
131
132
    
    log.debug('sample_set (%s)' % request.vars["id"])
    #if (auth.can_view_patient(request.vars["id"]) ):
    return dict(query=query,
133
                pre_process_list=pre_process_list,
134
135
136
                config_id=config_id,
                info=info_file,
                can_modify=auth.can_modify_sample_set(sample_set_id),
137
                can_upload=auth.can_upload_sample_set(sample_set_id),
138
139
140
141
142
143
                fused_count=fused_count,
                fused_file=fused_file,
                fused_filename=fused_filename,
                analysis_count=analysis_count,
                analysis_file = analysis_file,
                analysis_filename = analysis_filename,
144
                sample_type = db.sample_set[request.vars["id"]].sample_type,
145
146
                config=config,
                tag_decorator=tag_decorator)
147

148
149
150
## return a list of generic sample_sets
def all():
    start = time.time()
151
152
153
154
155
    if request.vars['type']:
        type = request.vars['type']
    else :
        type = defs.SET_TYPE_GENERIC

156
157
    if not auth.user :
        res = {"redirect" : URL('default', 'user', args='login', scheme=True, host=True,
158
                    vars=dict(_next=URL('sample_set', 'all', vars={'type': type, 'page': 0}, scheme=True, host=True)))
159
160
161
162
            }
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

    isAdmin = auth.is_admin()
163

164
165
166
167
168
169
170
    step = None
    page = None
    is_not_filtered = "sort" not in request.vars and "filter" not in request.vars
    if request.vars['page'] is not None and is_not_filtered:
        page = int(request.vars['page'])
        step = 50

171
172
173
174
175
    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
176
    group_ids = get_involved_groups()
177
178

    list = SampleSetList(type, page, step, tags=tags)
179
180
181
    list.load_creator_names()
    list.load_sample_information()
    list.load_config_information()
182
    if isAdmin or len(get_group_list(auth)) > 1:
183
        list.load_permitted_groups()
184
185
    list.load_anon_permissions()
    result = list.get_values()
186

187
188
    # failsafe if filtered display all results
    step = len(list) if step is None else step
189
    page = 0 if page is None else page
190

HERBERT Ryan's avatar
HERBERT Ryan committed
191
192
193
    factory = ModelFactory()
    helper = factory.get_instance(type=type)
    fields = helper.get_fields()
194
    sort_fields = helper.get_sort_fields()
195

196
197
198
199
    ##sort result
    reverse = False
    if request.vars["reverse"] == "true" :
        reverse = True
HERBERT Ryan's avatar
HERBERT Ryan committed
200
    if "sort" in request.vars:
201
        result = sorted(result, key = sort_fields[request.vars["sort"]]['call'], reverse=reverse)
202
203
204
    else:
        result = sorted(result, key = lambda row : row.id, reverse=not reverse)

205
206
    result = helper.filter(search, result)
    log.debug("%s list (%.3fs) %s" % (request.vars["type"], time.time()-start, search))
207
208
209


    return dict(query = result,
HERBERT Ryan's avatar
HERBERT Ryan committed
210
                fields = fields,
211
                helper = helper,
212
                group_ids = group_ids,
213
                isAdmin = isAdmin,
214
                reverse = reverse,
215
216
                step = step,
                page = page)
217

218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
def stats():
    start = time.time()
    if not auth.user :
        res = {"redirect" : URL('default', 'user', args='login', scheme=True, host=True,
                    vars=dict(_next=URL('sample_set', 'all', vars={'type': defs.SET_TYPE_PATIENT}, scheme=True, host=True)))
            }
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

    isAdmin = auth.is_admin()
    if request.vars['type']:
        type = request.vars['type']
    else :
        type = defs.SET_TYPE_GENERIC

    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
    group_ids = get_involved_groups()

    list = SampleSetList(type, tags=tags)
    list.load_sample_information()
    list.load_anon_permissions()
    result = list.get_values()

    factory = ModelFactory()
    helper = factory.get_instance(type=type)
    fields = helper.get_reduced_fields()

    ##sort result
    reverse = False
    if request.vars["reverse"] == "true" :
        reverse = True
    if "sort" in request.vars:
        result = sorted(result, key = lambda row : row[request.vars["sort"]], reverse=reverse)
    else:
        result = sorted(result, key = lambda row : row.id, reverse=not reverse)

    result = helper.filter(search, result)
    log.debug("%s stat list (%.3fs) %s" % (request.vars["type"], time.time()-start, search))

    return dict(query = result,
                fields = fields,
                helper = helper,
                group_ids = group_ids,
                isAdmin = isAdmin,
                reverse = False)

def result_files():
    from zipfile import ZipFile
    from cStringIO import StringIO
    import types
    errors = []
    config_id = request.vars['config_id']
    sample_set_ids = []
    if 'sample_set_ids' in request.vars:
        sample_set_ids = request.vars['sample_set_ids']

    #little hack since we can't pass array parameters with only one value
    if isinstance(sample_set_ids, types.StringTypes):
        sample_set_ids = [sample_set_ids]

281
282
283
284
285
286
    if int(config_id) == -1:
        config_query = (db.results_file.config_id > 0)
    else:
        config_query = (db.results_file.config_id == config_id)


287
288
289
290
291
292
293
294
    left_join = [
        db.patient.on(db.patient.sample_set_id == db.sample_set.id),
        db.run.on(db.run.sample_set_id == db.sample_set.id),
        db.generic.on(db.generic.sample_set_id == db.sample_set.id)
    ]
    q = db(
            (db.sample_set.id.belongs(sample_set_ids)) &
            (db.sample_set_membership.sample_set_id == db.sample_set.id) &
295
296
            (db.sequence_file.id == db.sample_set_membership.sequence_file_id) &
            (db.results_file.sequence_file_id == db.sequence_file.id) &
297
            (db.results_file.data_file != None) &
298
            config_query
299
300
        )

301
    results = q.select(db.results_file.ALL, db.sequence_file.ALL, db.sample_set.ALL, db.patient.ALL, db.run.ALL, db.generic.ALL, left=left_join)
302
303
304
305
306
307
308
309
310

    sample_types = ['patient', 'run', 'generic']
    mf = ModelFactory()
    helpers = {}

    for t in sample_types:
        helpers[t] = mf.get_instance(type=t)

    filename = "export_%s_%s.zip" % ('-'.join(sample_set_ids), str(datetime.date.today()))
311
312
313
314
315
316
317
318
319
    filedir = defs.DIR_SEQUENCES + '/' + filename
    try:
        zipfile = ZipFile(filedir, 'w')
        metadata = []
        for res in results:
            metadata.append({'id': res.sample_set.id,
                'name': helpers[res.sample_set.sample_type].get_name(res[res.sample_set.sample_type]),
                'file': res.results_file.data_file,
                'set_info': res[res.sample_set.sample_type].info,
320
321
                'sample_info': res.sequence_file.info,
                'sequence_file': res.sequence_file.filename})
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
            path = defs.DIR_RESULTS + res.results_file.data_file
            zipfile.writestr(res.results_file.data_file, open(path, 'rb').read())

        zipfile.writestr('metadata.json', json.dumps(metadata))

        zipfile.close()

        response.headers['Content-Type'] = "application/zip"
        response.headers['Content-Disposition'] = 'attachment; filename=%s' % filename# to force download as attachment

        return response.stream(open(filedir), chunk_size=4096)
    except:
        res = {"message": "an error occurred"}
        log.error("An error occured when creating archive of sample_sets %s" % str(sample_set_ids))
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    finally:
        try:
            os.unlink(filedir)
        except OSError:
            pass
342

343

344
345
## Stats

346
def mystats():
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
    start = time.time()

    # d = all()
    d = custom()
    
    # Build .vidjil file list
    f_samples = []
    for row in d['query']:
        found = {}
        f_results = defs.DIR_RESULTS + row.results_file.data_file

        f_fused = None
        pos_in_fused = None

        fused_file = ''
        # TODO: fix the following request
        # fused_file = db((db.fused_file.sample_set_id == row.sample_set.id) & (db.fused_file.config_id == row.results_file.config_id)).select(orderby = ~db.fused_file.id, limitby=(0,1))
        if len(fused_file) > 0 and fused_file[0].sequence_file_list is not None:
            sequence_file_list = fused_file[0].sequence_file_list.split('_')
            try:
                pos_in_fused = sequence_file_list.index(str(row.sequence_file.id))
                f_fused = defs.DIR_RESULTS + fused_file[0].fused_file
            except ValueError:
                pass
                
        metadata = { } # 'patient': row.patient, 'sequence_file': row.sequence_file }
        f_samples += [(metadata, f_results, f_fused, pos_in_fused)]
    
    # Send to vidjil_utils.stats
    res = vidjil_utils.stats(f_samples)
    d = {}
    d['stats'] = res
    d['f_samples'] = f_samples # TMP, for debug

    # Return
    log.debug("stats (%.3fs) %s" % (time.time()-start, request.vars["filter"]))
    return gluon.contrib.simplejson.dumps(d, separators=(',',':'))

Ryan Herbert's avatar
Ryan Herbert committed
385
386
387
388
389
390
## return form to create new set
def form():
    denied = False
    # edit set
    if("id" in request.vars):
        sample_set = db.sample_set[request.vars["id"]]
391
392
        set_type = sample_set.sample_type
        sset = db(db[set_type].sample_set_id == sample_set.id).select().first()
393
394
        if(auth.can_modify_sample_set(sset.sample_set_id)):
            groups = [get_set_group(sset.sample_set_id)]
395
            action = 'edit'
Ryan Herbert's avatar
Ryan Herbert committed
396
397
398
399
400
401
402
403
404
            max_group = None
        else:
            denied = True

    # new set
    elif (auth.can_create_patient()):
        sset = None
        set_type = request.vars["type"]
        creation_group_tuple = get_default_creation_group(auth)
405
406
        groups = creation_group_tuple[0]
        max_group = creation_group_tuple[1]
407
        action = 'add'
408
    else :
Ryan Herbert's avatar
Ryan Herbert committed
409
410
411
        denied = True

    if denied:
412
413
414
415
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

416
    message = '%s %s' % (action, set_type)
Ryan Herbert's avatar
Ryan Herbert committed
417
418
419
420
421
422
423
424
425
    sets = {
            'patient': [],
            'run': [],
            'generic': []
            }
    # We add a None object to the desired set type to initialise an empty form in the template.
    sets[set_type].append(sset)
    return dict(message=T(message),
                groups=groups,
426
                group_ids = get_involved_groups(),
Ryan Herbert's avatar
Ryan Herbert committed
427
                master_group=max_group,
428
429
                sets=sets,
                isEditing = (action=='edit'))
Ryan Herbert's avatar
Ryan Herbert committed
430

431
432


Ryan Herbert's avatar
Ryan Herbert committed
433
434
435
## create a patient if the html form is complete
## need ["first_name", "last_name", "birth_date", "info"]
## redirect to patient list if success
436
## return a flash error message if fail
Ryan Herbert's avatar
Ryan Herbert committed
437
438
439
def submit():
    data = json.loads(request.vars['data'], encoding='utf-8')
    mf = ModelFactory()
440

Ryan Herbert's avatar
Ryan Herbert committed
441
442
    error = False
    set_types = vidjil_utils.get_found_types(data)
443
444

    length_mapping = {}
445
    sum_sets = 0
Ryan Herbert's avatar
Ryan Herbert committed
446
447
    for set_type in set_types:
        helper = mf.get_instance(set_type)
448
        length = len(data[set_type])
449
        sum_sets += length
450
451
        if length not in length_mapping:
            length_mapping[length] = set_type
Ryan Herbert's avatar
Ryan Herbert committed
452
        for p in data[set_type]:
453
            errors = helper.validate(p)
454
            action = "add"
455
456
            if len(errors) > 0:
                p['error'] = errors
Ryan Herbert's avatar
Ryan Herbert committed
457
458
                error = True
                continue
459

Ryan Herbert's avatar
Ryan Herbert committed
460
461
            register = False
            reset = False
HERBERT Ryan's avatar
HERBERT Ryan committed
462

Ryan Herbert's avatar
Ryan Herbert committed
463
            name = helper.get_name(p)
464

Ryan Herbert's avatar
Ryan Herbert committed
465
            # edit
466
            if (p['sample_set_id'] != "" and auth.can_modify_sample_set(p['sample_set_id'])):
Ryan Herbert's avatar
Ryan Herbert committed
467
                reset = True
468
469
                sset = db(db[set_type].sample_set_id == p['sample_set_id']).select().first()
                db[set_type][sset.id] = p
470
                id_sample_set = sset['sample_set_id']
471

Ryan Herbert's avatar
Ryan Herbert committed
472
                if (sset.info != p['info']):
473
                    group_id = get_set_group(id_sample_set)
Ryan Herbert's avatar
Ryan Herbert committed
474
475
                    register = True
                    reset = True
476

477
                action = "edit"
478

Ryan Herbert's avatar
Ryan Herbert committed
479
480
            # add
            elif (auth.can_create_patient()):
481

Ryan Herbert's avatar
Ryan Herbert committed
482
                id_sample_set = db.sample_set.insert(sample_type=set_type)
483

Ryan Herbert's avatar
Ryan Herbert committed
484
485
486
                p['creator'] = auth.user_id
                p['sample_set_id'] = id_sample_set
                p['id'] = db[set_type].insert(**p)
487

Ryan Herbert's avatar
Ryan Herbert committed
488
                group_id = int(data["group"])
489

Ryan Herbert's avatar
Ryan Herbert committed
490
                register = True
491

Ryan Herbert's avatar
Ryan Herbert committed
492
                #patient creator automaticaly has all rights
493
                auth.add_permission(group_id, PermissionEnum.access.value, 'sample_set', p['sample_set_id'])
Ryan Herbert's avatar
Ryan Herbert committed
494

495
                action = "add"
Ryan Herbert's avatar
Ryan Herbert committed
496
497
498
499
500
501
502
503
504
505

                if (p['id'] % 100) == 0:
                    mail.send(to=defs.ADMIN_EMAILS,
                    subject="[Vidjil] %d" % p['id'],
                    message="The %dth %s has just been created." % (p['id'], set_type))

            else :
                p['error'].append("permission denied")
                error = True

506
            p['message'] = []
507
            mes = u"%s (%s) %s %sed" % (set_type, id_sample_set, name, action)
Ryan Herbert's avatar
Ryan Herbert committed
508
            p['message'].append(mes)
509
            log.info(mes, extra={'user_id': auth.user.id, 'record_id': p['id'], 'table_name': 'patient'})
Ryan Herbert's avatar
Ryan Herbert committed
510
511
512
513
            if register:
                register_tags(db, set_type, p["id"], p["info"], group_id, reset=reset)

    if not error:
514
515
516
517
        max_num = max(length_mapping.keys())
        msg = "successfully added/edited set(s)"
        if sum_sets == 1:
            res = {"redirect": "sample_set/index",
518
                   "args": { "id":  data[length_mapping[max_num]][0]['sample_set_id']},
519
520
521
                   "message": msg}
        else:
            res = {"redirect": "sample_set/all",
522
                    "args" : { "type" : length_mapping[max_num], "page": 0 },
523
                    "message": msg}
524
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
Ryan Herbert's avatar
Ryan Herbert committed
525
526
527
528
529
530
    else:
        sets = {
                'patient': data['patient'] if 'patient' in data else [],
                'run': data['run'] if 'run' in data else [],
                'generic': data['generic'] if 'generic' in data else []
                }
531
        response.view = 'sample_set/form.html'
Ryan Herbert's avatar
Ryan Herbert committed
532
533
534
        return dict(message=T("an error occured"),
                groups=[{'name': 'foobar', 'id': int(data['group'])}],
                master_group=data['group'],
535
536
                sets=sets,
                isEditing = (action=='edit'))
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557

def custom():
    start = time.time()

    if request.vars["config_id"] and request.vars["config_id"] != "-1" :
        config_id = long(request.vars["config_id"])
        config_name = db.config[request.vars["config_id"]].name
        config = True
        
    else:
        request.vars["config_id"] = -1
        config_id = -1
        config_name = None
        config = False
        
    if "custom_list" not in request.vars :
        request.vars["custom_list"] = []
    if type(request.vars["custom_list"]) is str :
        request.vars["custom_list"] = [request.vars["custom_list"]]
        
    myGroupBy = None
558
    helper = None
559
    if request.vars["id"] and auth.can_view_sample_set(request.vars["id"]):
560
561
562
        sample_set = db.sample_set[request.vars["id"]]
        factory = ModelFactory()
        helper = factory.get_instance(type=sample_set.sample_type)
563
        qq = (db.sample_set.id == request.vars["id"])
564
565
        
    else:
566
        qq = (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set))
567
        myGroupBy = db.sequence_file.id|db.patient.id|db.run.id|db.generic.id|db.results_file.config_id
568

569
570
571
572
573
574
575
576
    q = (qq
        & (auth.vidjil_accessible_query(PermissionEnum.read_config.value, db.config))
        & (db.sample_set_membership.sample_set_id == db.sample_set.id)
        & (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
        & (db.results_file.sequence_file_id==db.sequence_file.id)
        & (db.results_file.data_file != '')
        & (db.config.id==db.results_file.config_id))

577
578
    group_ids = get_involved_groups()

579
580
581
582
583
    ##filter
    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
584

585
586
587
588
589
590
591
    left_join = [
        db.patient.on(db.patient.sample_set_id == db.sample_set.id),
        db.run.on(db.run.sample_set_id == db.sample_set.id),
        db.generic.on(db.generic.sample_set_id == db.sample_set.id)
    ]

    select = [
592
        db.patient.id, db.patient.sample_set_id, db.patient.info, db.patient.first_name, db.patient.last_name,
593
594
595
596
597
598
599
600
        db.run.id, db.run.info, db.run.name,
        db.generic.id, db.generic.info, db.generic.name,
        db.results_file.id, db.results_file.config_id, db.sequence_file.sampling_date,
        db.sequence_file.pcr, db.config.name, db.results_file.run_date, db.results_file.data_file, db.sequence_file.filename,
        db.sequence_file.data_file, db.sequence_file.id, db.sequence_file.info,
        db.sequence_file.size_file
    ]

601
602
603
    if (tags is not None and len(tags) > 0):
        q = filter_by_tags(q, 'sequence_file', tags)
        count = db.tag.name.count()
604
        select = select + [db.tag_ref.record_id,
605
                db.tag_ref.table_name,
606
607
608
609
610
                count]

        query = db(q).select(
                *select,
                left = left_join,
611
612
613
614
615
616
617
                orderby = db.sequence_file.id|db.results_file.run_date,
                groupby = db.tag_ref.table_name|db.tag_ref.record_id,
                having = count >= len(tags)
            )

    else:
        query = db(q).select(
618
619
                *select,
                left = left_join,
620
                orderby = db.sequence_file.id|db.results_file.run_date,
621
622
623
624
625
626
627
628
                groupby = myGroupBy
            )

    for row in query :
        row.checked = False
        if (str(row.results_file.id) in request.vars["custom_list"]) :
            row.checked = True

629
        if row.patient.id is not None:
Ryan Herbert's avatar
Ryan Herbert committed
630
            #TODO use helper.
631
            row.names = vidjil_utils.display_names(row.patient.sample_set_id, row.patient.first_name, row.patient.last_name)
632
            info = row.patient.info
633
        elif row.run.id is not None:
634
635
            row.names = row.run.name
            info = row.run.info
636
637
638
        elif row.generic.id is not None:
            row.names = row.generic.name
            info = row.generic.info
639
        row.string = [row.names, row.sequence_file.filename, str(row.sequence_file.sampling_date), str(row.sequence_file.pcr), str(row.config.name), str(row.results_file.run_date), info]
640
    query = query.find(lambda row : ( vidjil_utils.advanced_filter(row.string,search) or row.checked) )
641
642
643
644
645

    
    if config :
        query = query.find(lambda row : ( row.results_file.config_id==config_id or (str(row.results_file.id) in request.vars["custom_list"])) )
    
646
    tag_decorator = TagDecorator(get_tag_prefix())
647
    log.debug("sample_set/custom (%.3fs) %s" % (time.time()-start, search))
648
649
650

    return dict(query=query,
                config_id=config_id,
651
                config=config,
652
                helper=helper,
653
654
                tag_decorator=tag_decorator,
                group_ids=group_ids)
655

656
def getStatHeaders():
657
    m = StatDecorator()
658
659
    b = BooleanDecorator()
    p = BarDecorator()
660
661
662
663
664
665
666
667
668
669
670
    bc = BarChartDecorator()
    return [('set_id', 'db', m),
            ('set_name', 'db', m),
            ('set_info', 'db', m),
            ('main_clone', 'parser', m),
            ('reads', 'parser', m),
            ('mapped', 'parser', m),
            ('mapped_percent', 'parser', p),
            ('bool', 'parser', b),
            ('bool_true', 'parser', b),
        ]
671

672
def getResultsFileStats(file_name, dest):
673
674
675
676
677
678
679
680
    file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
    with open(file_path, 'rb') as f:
        objects = ijson.items(f, 'samples.results_file_id')

        dest['results_file_ids'] = json.loads(mjson)['results_file_id']
        return dest

def getFusedStats(file_name, res, dest):
681
682
683
    file_path = "%s%s" % (defs.DIR_RESULTS, file_name)
    parser = VidjilParser()
    parser.addPrefix('clones.item', 'clones.item.top', operator.eq, 1)
684
685
    parser.addPrefix("reads")
    parser.addPrefix("samples")
686
687

    mjson = parser.extract(file_path)
688
689
690
691
692
693
694
695
696
    data = json.loads(mjson)
    result_index = -1
    if "results_file_id" in data['samples']:
        result_index = data['samples']['results_file_id'].index(res['resuts_file_id'])
    elif "original_names" in data['samples']:
        result_index = data['samples']['original_names'].index(defs.DIR_SEQUENCES + res['sequence_file'])
    dest['main_clone'] = data['clones'][0]['name']
    reads = data['reads']['total'][result_index]
    dest['reads'] = reads
697
698
699
700
    dest['mapped'] = "%d" % (data['reads']['segmented'][result_index])
    dest['mapped_percent'] = 100.0 * (float(data['reads']['segmented'][result_index])/float(reads))
    dest['bool'] = False
    dest['bool_true'] = True
701
702
    return dest

703

704
705
706
707
708
709
710
711
712
def getStatData(results_file_ids):
    mf = ModelFactory()
    set_types = [defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN, defs.SET_TYPE_GENERIC]
    helpers = {}
    for stype in set_types:
        helpers[stype] = mf.get_instance(stype)

    query = db(
        (db.results_file.id.belongs(results_file_ids)) &
713
714
        (db.sequence_file.id == db.results_file.sequence_file_id) &
        (db.sample_set_membership.sequence_file_id == db.sequence_file.id) &
715
        (db.sample_set.id == db.sample_set_membership.sample_set_id) &
716
717
718
        (db.config.id == db.results_file.config_id) &
        (db.fused_file.config_id == db.config.id) &
        (db.fused_file.sample_set_id == db.sample_set.id)
719
        ).select(
720
            db.results_file.data_file.with_alias("results_file"), db.results_file.id.with_alias("results_file_id"),
721
            db.fused_file.fused_file.with_alias("fused_file"),
722
            db.sequence_file.data_file.with_alias("sequence_file"),
723
724
            db.sample_set.id.with_alias("set_id"),
            db.sample_set.sample_type.with_alias("sample_type"),
725
            db.patient.first_name, db.patient.last_name, db.patient.info.with_alias('set_info'), db.patient.sample_set_id,
726
727
728
            db.run.name,
            db.generic.name,
            db.config.name,
729
730

            db.generic.name.with_alias("set_name"), # use generic name as failsafe for set name
731
732
733
734
735
736
737
738
739
740
            left = [
                db.patient.on(db.patient.sample_set_id == db.sample_set.id),
                db.run.on(db.run.sample_set_id == db.sample_set.id),
                db.generic.on(db.generic.sample_set_id == db.sample_set.id)
            ]
        )

    data = []
    for res in query:
        d = {}
741
742
        set_type = res.sample_type
        headers = getStatHeaders()
743
        d = getFusedStats(res.fused_file, res, d)
744
        for head, htype, model in headers:
745
            if htype == 'db':
746
747
748
                d[head] = res[head]
            d[head] = model.decorate(d[head])
            log.debug("%s: %s" % (head, d[head]))
749
        d['set_name'] = helpers[set_type].get_name(res[set_type])
750
751
752
753
754
        data.append(d)
    return data

def multi_sample_stats():
    data = {}
755
    data['headers'] = [h for h, t, m in getStatHeaders()]
756
    results = []
757
758
759
760
    custom_result = request.vars['custom_result']
    if not isinstance(custom_result, list):
        custom_result = [custom_result]

761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
    custom_result = [long(i) for i in custom_result]

    permitted_results = db(
        (auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set)) &
        (db.sample_set.id == db.sample_set_membership.sample_set_id) &
        (db.sample_set_membership.sequence_file_id == db.results_file.sequence_file_id) &
        (db.results_file.id.belongs(custom_result))
    ).select(
            db.results_file.id.with_alias('results_file_id')
        )

    permitted_results_ids = [r.results_file_id for r in permitted_results]
    log.debug("premitted: " + str(permitted_results_ids))
    log.debug("custom: " + str(custom_result))
    if set(permitted_results_ids) != set(custom_result):
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

780
    results = getStatData(custom_result)
781
782
783
    data['results'] = results
    return dict(data=data)

784
785
def confirm():
    if auth.can_modify_sample_set(request.vars["id"]):
786
787
788
789
        sample_set = db.sample_set[request.vars["id"]]
        data = db(db[sample_set.sample_type].sample_set_id == sample_set.id).select().first()
        factory = ModelFactory()
        helper = factory.get_instance(type=sample_set.sample_type)
790
791
        log.debug('request sample_set deletion')
        return dict(message=T('confirm sample_set deletion'),
792
793
                    data=data,
                    helper=helper)
794
795
796
797
798
799
800
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

def delete():
    if (auth.can_modify_sample_set(request.vars["id"]) ):
801
        sample_set = db.sample_set[request.vars["id"]]
802
        sample_type = sample_set.sample_type
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
        if sample_set is None:
            res = {"message": 'An error occured. This sample_set may have already been deleted'}
            log.error(res)
            return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

        #delete data file
        query = db( (db.sample_set_membership.sample_set_id == sample_set.id)
                    & (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
                ).select(db.sequence_file.id)
        for row in query :
            db(db.results_file.sequence_file_id == row.id).delete()

        #delete sequence file
        query = db((db.sequence_file.id == db.sample_set_membership.sequence_file_id)
            & (db.sample_set_membership.sample_set_id == sample_set.id)
            ).select(db.sequence_file.id)
        for row in query :
            db(db.sequence_file.id == row.id).delete()

        #delete patient sample_set
        db(db.sample_set.id == sample_set.id).delete()

        res = {"redirect": "sample_set/all",
826
               "args": {"type": sample_type, "page": 0},
827
828
829
830
831
832
833
834
               "success": "true",
               "message": "sample set ("+str(request.vars["id"])+") deleted"}
        log.info(res, extra={'user_id': auth.user.id, 'record_id': request.vars["id"], 'table_name': 'sample_set'})
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905

#
def permission():
    if (auth.can_modify_sample_set(request.vars["id"]) ):
        sample_set = db.sample_set[request.vars["id"]]
        stype = sample_set.sample_type
        factory = ModelFactory()
        helper = factory.get_instance(type=stype)

        data = db(db[stype].sample_set_id == sample_set.id).select().first()

        query = db( db.auth_group.role != 'admin' ).select()

        for row in query :
            row.owner = row.role
            if row.owner[:5] == "user_" :
                id = int(row.owner[5:])
                row.owner = db.auth_user[id].first_name + " " + db.auth_user[id].last_name

            permissions = db(
                    (db.auth_permission.group_id == row.id) &
                    (db.auth_permission.record_id == 0) &
                    (db.auth_permission.table_name == db.sample_set)).select()
            row.perms = ', '.join(map(lambda x: x.name, permissions))

            row.parent_access = ', '.join(str(value) for value in auth.get_access_groups(db[stype], request.vars['id'], group=row.id))
            row.read =  auth.get_group_access(sample_set.sample_type, data.id, row.id)

        return dict(query=query,
                    helper=helper,
                    data=data)
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))

#
def change_permission():
    if (auth.can_modify_sample_set(request.vars["sample_set_id"]) ):
        ssid = request.vars["sample_set_id"]
        sample_set = db.sample_set[ssid]
        sample_type = sample_set.sample_type
        data_id = db(db[sample_type].sample_set_id == ssid).select().first().id

        error = ""
        if request.vars["group_id"] == "" :
            error += "missing group_id, "
        if ssid == "" :
            error += "missing sample_set_id, "

        if error=="":
            if auth.get_group_access(sample_type,
                      data_id,
                      int(request.vars["group_id"])):
                auth.del_permission(request.vars["group_id"], PermissionEnum.access.value, db[sample_type], data_id)
                res = {"message" : "access '%s' deleted to '%s'" % (PermissionEnum.access.value, db.auth_group[request.vars["group_id"]].role)}
            else :

                auth.add_permission(request.vars["group_id"], PermissionEnum.access.value, db[sample_type], data_id)
                res = {"message" : "access '%s' granted to '%s'" % (PermissionEnum.access.value, db.auth_group[request.vars["group_id"]].role)}

            log.info(res)
            return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
        else :
            res = {"message": "incomplete request : "+error }
            log.error(res)
            return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
    else :
        res = {"message": ACCESS_DENIED}
        log.error(res)
        return gluon.contrib.simplejson.dumps(res, separators=(',',':'))
906

907
def get_sample_set_list(stype, q):
Ryan Herbert's avatar
Ryan Herbert committed
908
909
910
911
    factory = ModelFactory()
    helper = factory.get_instance(type=stype)

    limitby = None
912
    if q is not None and len(q) == 0:
913
        q = None
914
    if not q :
Ryan Herbert's avatar
Ryan Herbert committed
915
916
        limitby = (0, 10)

917
    filter_query = helper.get_name_filter_query(q)
918
    query = db(
919
        (auth.vidjil_accessible_query(PermissionEnum.admin.value, db.sample_set))&
Ryan Herbert's avatar
Ryan Herbert committed
920
921
        (db[stype].sample_set_id == db.sample_set.id) &
        (filter_query)
922
    ).select(
Ryan Herbert's avatar
Ryan Herbert committed
923
924
925
        db[stype].ALL, # sub optimal, use helpers to reduce ?
        orderby = ~db[stype].sample_set_id,
        limitby = limitby
926
927
928
929
930
    )
    ss_list = []

    for row in query :
        tmp = helper.get_id_string(row)
931
        ss_list.append({'name':tmp, 'id': row.sample_set_id, 'type': stype})
932
933
934
935
936
937
    return ss_list

def auto_complete():
    if "keys" not in request.vars:
        return error_message("missing group ids")

938
    query = json.loads(request.vars['keys'])[0]
Ryan Herbert's avatar
Ryan Herbert committed
939
940
    sample_types = [defs.SET_TYPE_PATIENT, defs.SET_TYPE_RUN, defs.SET_TYPE_GENERIC]
    result = []
941
    for sample_type in sample_types:
Ryan Herbert's avatar
Ryan Herbert committed
942
        result += get_sample_set_list(sample_type, query)
943

944
945
946
    res = {}
    res[query] = result
    return json.dumps(res)