Attention une mise à jour du serveur va être effectuée le lundi 17 mai entre 13h et 13h30. Cette mise à jour va générer une interruption du service de quelques minutes.

my_account.py 11.5 KB
Newer Older
Ryan Herbert's avatar
Ryan Herbert committed
1 2 3
# coding: utf8

from datetime import datetime, timedelta
Ryan Herbert's avatar
Ryan Herbert committed
4
import time
5
import types
Ryan Herbert's avatar
Ryan Herbert committed
6 7 8 9 10 11 12 13

if request.env.http_origin:
    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Max-Age'] = 86400

ACCESS_DENIED = "access denied"

14 15
def generic_get_group(element_type):
    group = {}
Ryan Herbert's avatar
Ryan Herbert committed
16 17
    separator = "|| ';' ||"
    fields = ["config.name",
18 19
              "%s_file.sample_set_id" % element_type,
              "%s_file.config_id" % element_type,
Ryan Herbert's avatar
Ryan Herbert committed
20 21 22
              "sample_set.sample_type"
            ]
    group_concat = "GROUP_CONCAT(DISTINCT  " + separator.join(fields)
23 24 25
    group['patient'] =  group_concat + separator + "patient.first_name || ' ' || patient.last_name || '#')"
    group['run'] = group_concat + separator + "run.name || '#')"
    group['set'] = group_concat + separator + "generic.name || '#')"
26 27 28 29
    return group

def get_group_fuses():
    return generic_get_group('fused')
Ryan Herbert's avatar
Ryan Herbert committed
30

Ryan Herbert's avatar
Ryan Herbert committed
31 32 33 34 35 36 37
def get_group_analyses():
    group = {}
    separator = "|| ';' ||"
    fields = ["analysis_file.sample_set_id",
              "sample_set.sample_type"
            ]
    group_concat = "GROUP_CONCAT(DISTINCT  " + separator.join(fields)
38 39 40
    group['patient'] =  group_concat + separator + "patient.first_name || ' ' || patient.last_name || '#')"
    group['run'] = group_concat + separator + "run.name || '#')"
    group['set'] = group_concat + separator + "generic.name || '#')"
Ryan Herbert's avatar
Ryan Herbert committed
41 42
    return group

Ryan Herbert's avatar
Ryan Herbert committed
43 44 45 46 47
def group_permissions():
    return "GROUP_CONCAT(DISTINCT auth_permission.name)"

def base_query(group_list):
    return (db.auth_group.id.belongs(group_list) &
48
        (db.auth_permission.group_id == db.auth_group.id))
Ryan Herbert's avatar
Ryan Herbert committed
49

50 51 52 53 54 55
def access_query(group_list):
    return (base_query(group_list) &
            (db.auth_permission.table_name == 'sample_set') &
            (db.auth_permission.name == 'access') &
            (db.auth_permission.record_id > 0))

56 57 58 59 60 61 62
def filter_by_tags(tags):
        return ((db.tag.name.upper().belongs([t.upper() for t in tags])) &
        (db.tag_ref.tag_id == db.tag.id) &
        (db.tag_ref.table_name == 'sequence_file') &
        (db.tag_ref.record_id == db.sample_set_membership.sequence_file_id)
    )

Ryan Herbert's avatar
Ryan Herbert committed
63 64 65 66 67 68 69
def base_left():
    return [db.sample_set.on(
                db.sample_set.id == db.auth_permission.record_id),
            db.sample_set_membership.on(
                db.sample_set_membership.sample_set_id == db.sample_set.id)]

def get_permissions(group_list):
70
    return db(db.auth_group.id.belongs(group_list)
Ryan Herbert's avatar
Ryan Herbert committed
71 72 73
        ).select(
            db.auth_group.role,
            group_permissions(),
74 75 76 77
            left=(db.auth_permission.on(
                (db.auth_permission.group_id == db.auth_group.id) &
                (db.auth_permission.table_name == 'sample_set') &
                (db.auth_permission.record_id == 0))),
Ryan Herbert's avatar
Ryan Herbert committed
78 79 80
            groupby=db.auth_group.role
        )

81
def get_most_used_tags(group_list):
Ryan Herbert's avatar
Ryan Herbert committed
82 83
    left = [
        db.sample_set_membership.on(
84
            db.sample_set_membership.sample_set_id == db.auth_permission.record_id),
Ryan Herbert's avatar
Ryan Herbert committed
85 86 87 88 89 90 91 92 93
        db.tag_ref.on(
            (db.tag_ref.table_name == 'sequence_file') &
            (db.tag_ref.record_id == db.sample_set_membership.sequence_file_id)),
        db.tag.on(
            db.tag_ref.tag_id == db.tag.id)
    ]

    return db(base_query(group_list)).select(
        db.auth_group.role,
94 95
        db.tag_ref.id.count().with_alias('count'),
        db.tag.name,
Ryan Herbert's avatar
Ryan Herbert committed
96
        left=left,
97 98 99
        groupby=(db.auth_group.role, db.tag.name),
        orderby=~db.tag_ref.id.count(),
        limitby=(0,100)
Ryan Herbert's avatar
Ryan Herbert committed
100 101
    )

Ryan Herbert's avatar
Ryan Herbert committed
102
def index():
Ryan Herbert's avatar
Ryan Herbert committed
103
    start = time.time()
Ryan Herbert's avatar
Ryan Herbert committed
104

105
    since = datetime.today() - timedelta(days=30)
Ryan Herbert's avatar
Ryan Herbert committed
106

107
    if auth.is_admin() and 'data' in request.vars:
108
        import json
109
        group_list = json.loads(request.vars['data'])['group_ids']
Ryan Herbert's avatar
Ryan Herbert committed
110 111 112
    else:
        group_list = [int(g.id) for g in auth.get_user_groups() + auth.get_user_group_parents()]

Ryan Herbert's avatar
Ryan Herbert committed
113
    log.debug("group_list: %s" % group_list)
Ryan Herbert's avatar
Ryan Herbert committed
114 115 116 117 118

    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])
Ryan Herbert's avatar
Ryan Herbert committed
119

120
    result = {}
Ryan Herbert's avatar
Ryan Herbert committed
121
    perm_query = get_permissions(group_list)
122 123
    for r in perm_query:
        if(r.auth_group.role not in result):
Ryan Herbert's avatar
Ryan Herbert committed
124 125
            result[r.auth_group.role] = {}
            for set_type in ['patient', 'run', 'set']:
Ryan Herbert's avatar
Ryan Herbert committed
126
                result[r.auth_group.role][set_type] = {'count': {'num_sets': 0, 'num_samples': 0, 'sample_type': 'generic' if set_type == 'set' else set_type}}
Ryan Herbert's avatar
Ryan Herbert committed
127
                result[r.auth_group.role][set_type]['tags'] = []
128
                result[r.auth_group.role][set_type]['statuses'] = ""
Ryan Herbert's avatar
Ryan Herbert committed
129
                result[r.auth_group.role][set_type]['num_jobs'] = 0
130
            result[r.auth_group.role]['fuses'] = []
Ryan Herbert's avatar
Ryan Herbert committed
131
            result[r.auth_group.role]['analyses'] = []
Ryan Herbert's avatar
Ryan Herbert committed
132
            result[r.auth_group.role]['tags'] = []
133
        result[r.auth_group.role]['permissions'] = "" if r._extra[group_permissions()] is None else r._extra[group_permissions()]
134

135
    query = access_query(group_list)
Ryan Herbert's avatar
Ryan Herbert committed
136
    if (tags is not None and len(tags) > 0):
137
        query = (query & filter_by_tags(tags))
Ryan Herbert's avatar
Ryan Herbert committed
138

139
    group_statuses = "GROUP_CONCAT(DISTINCT scheduler_task.id || ';' || scheduler_task.status || '#')"
Ryan Herbert's avatar
Ryan Herbert committed
140
    group_fuses = get_group_fuses()
Ryan Herbert's avatar
Ryan Herbert committed
141
    group_analyses = get_group_analyses()
142

Ryan Herbert's avatar
Ryan Herbert committed
143
    left = base_left() + [
144 145
        db.sequence_file.on(
            db.sequence_file.id == db.sample_set_membership.sequence_file_id),
Ryan Herbert's avatar
Ryan Herbert committed
146 147
        db.results_file.on(
            db.sample_set_membership.sequence_file_id == db.results_file.sequence_file_id),
148 149
        db.scheduler_task.on(
            (db.scheduler_task.id == db.results_file.scheduler_task_id) &
150
            (db.scheduler_task.start_time >= since)),
Ryan Herbert's avatar
Ryan Herbert committed
151 152
        db.fused_file.on(
            (db.fused_file.sample_set_id == db.sample_set.id) &
153
            (db.fused_file.fuse_date >= since)),
Ryan Herbert's avatar
Ryan Herbert committed
154
        db.config.on(
Ryan Herbert's avatar
Ryan Herbert committed
155 156 157 158
            db.config.id == db.fused_file.config_id),
        db.analysis_file.on(
            (db.analysis_file.sample_set_id == db.sample_set.id) &
            (db.analysis_file.analyze_date > since))
Ryan Herbert's avatar
Ryan Herbert committed
159 160
    ]

Ryan Herbert's avatar
Ryan Herbert committed
161 162
    select = [
        db.auth_group.id,
Ryan Herbert's avatar
Ryan Herbert committed
163
        db.auth_group.role,
Ryan Herbert's avatar
Ryan Herbert committed
164 165 166
        db.sample_set.sample_type.with_alias('sample_type'),
        db.sample_set.id.count(distinct=True).with_alias('num_sets'),
        db.sample_set_membership.sequence_file_id.count(distinct=True).with_alias('num_samples'),
Ryan Herbert's avatar
Ryan Herbert committed
167
        group_statuses,
Ryan Herbert's avatar
Ryan Herbert committed
168 169
    ]

170
    queries = {}
171 172 173 174
    for set_type in ['patient', 'run', 'generic']:
        key = 'set' if set_type == 'generic' else set_type
        set_query = (query &
                    (db[set_type].sample_set_id == db.auth_permission.record_id))
Ryan Herbert's avatar
Ryan Herbert committed
175

176 177
        queries[key] = db(set_query).select(
            group_fuses[key],
Ryan Herbert's avatar
Ryan Herbert committed
178
            group_analyses[key],
179 180
            *select,
            left=left,
181 182
            groupby=(db.auth_group.role, db.sample_set.sample_type),
            orderby=~db.scheduler_task.start_time
183
        )
Ryan Herbert's avatar
Ryan Herbert committed
184

185
    list_size = 50
Ryan Herbert's avatar
Ryan Herbert committed
186
    for key in queries: # patient, run, set
Ryan Herbert's avatar
Ryan Herbert committed
187
        query = queries[key]
Ryan Herbert's avatar
Ryan Herbert committed
188
        for r in query: # 1 or 0 rows
Ryan Herbert's avatar
Ryan Herbert committed
189 190 191
            result[r.auth_group.role][key]['count']['num_sets'] += r.num_sets
            result[r.auth_group.role][key]['count']['num_samples'] += r.num_sets
            result[r.auth_group.role][key]['count']['sample_type'] = r.sample_type
192
            statuses = "" if r._extra[group_statuses] is None else "".join([s.strip('#').split(';')[1][0] for s in r._extra[group_statuses].strip(',').split(',') if s[-1] == '#'])
193 194
            result[r.auth_group.role][key]['num_jobs'] = len(statuses)
            result[r.auth_group.role][key]['statuses'] += statuses[:list_size]
Ryan Herbert's avatar
Ryan Herbert committed
195

196
            fuses = [] if r._extra[group_fuses[key]] is None else [fuse.strip('#').split(';') for fuse in r._extra[group_fuses[key]].strip(',').split(',') if fuse[-1] == '#']
197
            result[r.auth_group.role]['fuses'] += (fuses[:list_size])
Ryan Herbert's avatar
Ryan Herbert committed
198

199
            analyses = [] if r._extra[group_analyses[key]] is None else [analysis.strip('#').split(';') for analysis in r._extra[group_analyses[key]].strip(',').split(',') if analysis[-1] == '#']
Ryan Herbert's avatar
Ryan Herbert committed
200 201
            result[r.auth_group.role]['analyses'] += analyses[:list_size]

202
    tags = get_most_used_tags(group_list) # list tags used without filtering
Ryan Herbert's avatar
Ryan Herbert committed
203
    for r in tags:
204
        if(r.tag.name is not None):
205 206 207 208 209
            if r.count > 1:
                tag_string = "%s (%d)" % (r.tag.name, r.count)
            else:
                tag_string = "%s" % r.tag.name
            result[r.auth_group.role]['tags'].append(tag_string)
Ryan Herbert's avatar
Ryan Herbert committed
210

Ryan Herbert's avatar
Ryan Herbert committed
211
    involved_group_ids = get_involved_groups() # for search autocomplete
Ryan Herbert's avatar
Ryan Herbert committed
212

213 214
    keys = sorted(result.keys(), key=lambda x: (result[x]['patient']['count']['num_sets'] + result[x]['run']['count']['num_sets'] + result[x]['set']['count']['num_sets']), reverse=True)

Ryan Herbert's avatar
Ryan Herbert committed
215
    log.debug("my account list (%.3fs)" % (time.time()-start))
216 217
    return dict(keys = keys,
                result=result,
Ryan Herbert's avatar
Ryan Herbert committed
218 219
                group_ids = group_list,
                involved_group_ids = involved_group_ids)
Ryan Herbert's avatar
Ryan Herbert committed
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249

def jobs():
    since = datetime.today() - timedelta(days=30)

    if auth.is_admin() and 'group_ids' in request.vars and request.vars['group_ids'] is not None:
        group_list = request.vars['group_ids']
        if isinstance(group_list, types.StringTypes):
            group_list = [group_list]
    else:
        group_list = [int(g.id) for g in auth.get_user_groups() + auth.get_user_group_parents()]

    log.debug("group_list: %s" % group_list)

    if "filter" not in request.vars :
        request.vars["filter"] = ""

    search, tags = parse_search(request.vars["filter"])

    query = (access_query(group_list) &
            (db.sample_set.id == db.auth_permission.record_id) &
            (db.sample_set_membership.sample_set_id == db.sample_set.id) &
            (db.sequence_file.id == db.sample_set_membership.sequence_file_id) &
            (db.results_file.sequence_file_id == db.sequence_file.id) &
            (db.config.id == db.results_file.config_id) &
            (db.scheduler_task.id == db.results_file.scheduler_task_id) &
            (db.scheduler_task.start_time > since))

    if (tags is not None and len(tags) > 0):
        query = (query & filter_by_tags(tags))

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
    names = {}
    names['patient'] = "patient.first_name || ' ' || patient.last_name"
    names['run'] = "run.name"
    names['generic'] = "generic.name"

    queries = {}
    for set_type in ['patient', 'run', 'generic']:
        set_query = (query &
                    (db[set_type].sample_set_id == db.sample_set.id))

        key = 'set' if set_type == 'generic' else set_type
        queries[key] = db(set_query).select(
                names[set_type],
                db.sample_set.id.with_alias('sample_set_id'),
                db.sample_set.sample_type.with_alias('sample_type'),
                db.sequence_file.filename.with_alias('filename'),
                db.sequence_file.info.with_alias('info'),
                db.config.id.with_alias('config_id'),
                db.config.name.with_alias('config'),
                db.scheduler_task.status.with_alias('status'),
                db.scheduler_task.start_time.with_alias('time'),
                orderby=~db.scheduler_task.start_time
            )
Ryan Herbert's avatar
Ryan Herbert committed
273 274 275 276

    involved_group_ids = get_involved_groups() # for search autocomplete
    tagdecorator = TagDecorator(get_tag_prefix())

277 278 279 280 281 282 283
    result = []
    for key in queries:
        result += queries[key]

    sorted_start = time.time()
    result = sorted(result, key=lambda x: x.time, reverse=True)
    log.debug("jobs list sort (%.3fs)" % (time.time() - sorted_start))
Ryan Herbert's avatar
Ryan Herbert committed
284 285 286
    return dict(result=result,
                group_ids = group_list,
                involved_group_ids = involved_group_ids,
287 288
                tagdecorator = tagdecorator,
                names = names)