Merge branch 'feature-s/2951-my-account-stats' into 'dev'

My Account

Closes #2952, #4469, #4484, #2951, #3880, #2606, and #2605

See merge request !837

browser/css/dark.css

@@ -2304,3 +2304,30 @@ form .form_label {
 .list_lock_on {
   color: #839496;
 }
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+.set_group_header {
+  width: 100%;
+}
+.set_cell {
+  word-break: break-all;
+}
+.set_data {
+  width: 100%;
+}
+.analysis_button {
+  display: inline-flex;
+}
+.margined-top {
+  margin-top: 10px;
+}
+.margined-bottom {
+  margin-bottom: 10px;
+}

browser/css/light.css

@@ -2304,3 +2304,30 @@ form .form_label {
 .list_lock_on {
   color: #657b83;
 }
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+.set_group_header {
+  width: 100%;
+}
+.set_cell {
+  word-break: break-all;
+}
+.set_data {
+  width: 100%;
+}
+.analysis_button {
+  display: inline-flex;
+}
+.margined-top {
+  margin-top: 10px;
+}
+.margined-bottom {
+  margin-bottom: 10px;
+}

browser/css/segmenter_page.css

@@ -2304,6 +2304,33 @@ form .form_label {
 .list_lock_on {
   color: #657b83;
 }
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+.set_group_header {
+  width: 100%;
+}
+.set_cell {
+  word-break: break-all;
+}
+.set_data {
+  width: 100%;
+}
+.analysis_button {
+  display: inline-flex;
+}
+.margined-top {
+  margin-top: 10px;
+}
+.margined-bottom {
+  margin-bottom: 10px;
+}
 body {
   line-height: 20px;
 }

browser/css/svg.css

@@ -2303,6 +2303,33 @@ form .form_label {
 .list_lock_on {
   color: #777777;
 }
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+.set_group_header {
+  width: 100%;
+}
+.set_cell {
+  word-break: break-all;
+}
+.set_data {
+  width: 100%;
+}
+.analysis_button {
+  display: inline-flex;
+}
+.margined-top {
+  margin-top: 10px;
+}
+.margined-bottom {
+  margin-bottom: 10px;
+}
 #resolution5 {
   opacity: 0;
 }

browser/css/talk.css

@@ -2304,6 +2304,33 @@ form .form_label {
 .list_lock_on {
   color: #657b83;
 }
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+.set_group_header {
+  width: 100%;
+}
+.set_cell {
+  word-break: break-all;
+}
+.set_data {
+  width: 100%;
+}
+.analysis_button {
+  display: inline-flex;
+}
+.margined-top {
+  margin-top: 10px;
+}
+.margined-bottom {
+  margin-bottom: 10px;
+}
 #mid-container {
   font-size: 125%;
 }

browser/css/vidjil.less

@@ -2651,4 +2651,38 @@ form {
 }
 .list_lock_on {
   color: @default;
-}
\ No newline at end of file
+}
+
+.set_group {
+  display: inline-flex;
+  flex-wrap: wrap;
+  flex-direction: row;
+  margin-bottom: 25px;
+  margin-right: 15px;
+  flex-flow: column;
+  width: 48%;
+}
+
+.set_group_header {
+    width: 100%;
+}
+
+.set_cell {
+  word-break: break-all;
+}
+
+.set_data {
+  width: 100%;
+}
+
+.analysis_button {
+  display: inline-flex;
+}
+
+.margined-top {
+  margin-top: 10px;
+}
+
+.margined-bottom {
+  margin-bottom: 10px;
+}

browser/js/database.js

@@ -291,6 +291,34 @@ Database.prototype = {
         });
     },
 
+    callUrlJson : function(url, args) {
+        var self=this;
+
+        $.ajax({
+            type: "POST",
+            crossDomain: true,
+            url: url,
+            data: {'data': JSON.stringify(args)},
+            timeout: DB_TIMEOUT_CALL,
+            xhrFields: {withCredentials: true},
+            success: function (result) {
+                self.display_result(result, url, args)
+                self.connected = true;
+            },
+            error: function (request, status, error) {
+                self.connected = false;
+                if (status === "timeout") {
+                    console.log({"type": "flash", "default" : "database_timeout", "priority": 2});
+                } else {
+                    self.check_cert()
+                }
+                self.warn("callUrlJson: " + status + " - " + url.replace(self.db_address, '') + "?" + self.argsToStr(args))
+            }
+
+        });
+
+    },
+
     callLinkable: function (linkable) {
         var href = linkable.attr('href');
         var type = linkable.data('linkable-type');
@@ -1228,6 +1256,29 @@ Database.prototype = {
         this.updateStatsButton();
     },
 
+    callGroupStats: function() {
+        var group_ids = [];
+        $('[name^="group_ids"]:checked').each(function() {
+            group_ids.push($(this).val());
+        });
+        this.callUrlJson(DB_ADDRESS + 'my_account/index', {'group_ids': group_ids});
+    },
+
+    callJobStats: function() {
+        var group_ids = [];
+        $('[name^="group_ids"]:checked').each(function() {
+            group_ids.push($(this).val());
+        });
+        this.callUrlJson(DB_ADDRESS + 'my_account/jobs', {'group_ids': group_ids});
+    },
+
+    stopGroupPropagate: function(e) {
+        if(!e) {
+            e = window.event
+        }
+        e.stopPropagation();
+    },
+
     // Log functions, to server
     // 'quiet' is set to true to avoid infinite loops with timeouts
     log : function (lvl, msg) {

browser/test/functional/vidjil_browser.rb

@@ -373,6 +373,11 @@ class VidjilBrowser < Watir::Browser
     return tag_selector.button(:text => 'ok')
   end
 
+  # Enter in dev mode
+  def devel_mode
+    $b.execute_script('$(".devel-mode").show();')
+  end
+
   # Unselect everything, both on clones, and move away from menus (as if we click on the back)
   def unselect
     $b.execute_script('m.unselectAll()')

server/web2py/applications/vidjil/controllers/group.py

@@ -177,7 +177,80 @@ def info():
         log.info("access user list", extra={'user_id': auth.user.id,
                 'record_id': request.vars["id"],
                 'table_name': "group"})
-        return dict(message=T('group info'))
+
+        group = db.auth_group[request.vars["id"]]
+
+        base_query = ((db.auth_user.id == db.auth_membership.user_id)
+                & (db.auth_membership.group_id == request.vars["id"]))
+
+        parent_group = db(db.group_assoc.second_group_id == request.vars["id"]).select()
+
+        group_ids = [request.vars["id"]] + [r.first_group_id for r in parent_group]
+
+        base_left = [db.auth_permission.on(
+                        (db.auth_permission.group_id.belongs(group_ids)) &
+                        (db.auth_permission.name == 'access') &
+                        (db.auth_permission.table_name == 'sample_set') &
+                        (db.auth_permission.record_id > 0)),
+                    db.sample_set_membership.on(
+                        db.sample_set_membership.sample_set_id == db.auth_permission.record_id)]
+
+        query = db(base_query).select(
+                db.auth_user.id,
+                db.auth_user.first_name,
+                db.auth_user.last_name,
+                db.auth_user.email,
+                db.sequence_file.id.count(True).with_alias('file_count'),
+                db.sequence_file.size_file.coalesce_zero().sum().with_alias('size'),
+                left=base_left + [db.sequence_file.on(
+                        (db.sequence_file.provider == db.auth_user.id) &
+                        (db.sequence_file.id == db.sample_set_membership.sequence_file_id))
+                ],
+                groupby=(db.auth_user.id)
+            )
+
+        sset_count = db(base_query).select(
+                db.auth_user.id,
+                db.patient.id.count(True).with_alias('patient_count'),
+                db.run.id.count(True).with_alias('run_count'),
+                db.generic.id.count(True).with_alias('generic_count'),
+                left=base_left + [db.patient.on(
+                        (db.patient.creator == db.auth_user.id) &
+                        (db.patient.sample_set_id == db.sample_set_membership.sample_set_id)),
+                    db.run.on(
+                        (db.run.creator == db.auth_user.id) &
+                        (db.run.sample_set_id == db.sample_set_membership.sample_set_id)),
+                    db.generic.on(
+                        (db.generic.creator == db.auth_user.id) &
+                        (db.generic.sample_set_id == db.sample_set_membership.sample_set_id))
+                ],
+                groupby=(db.auth_user.id)
+            )
+
+        logins = db(base_query).select(
+                db.auth_user.id,
+                db.auth_event.time_stamp.max().with_alias('last_login'),
+                left=[db.auth_event.on(
+                    (db.auth_event.user_id == db.auth_user.id) &
+                    (db.auth_event.origin == 'auth') &
+                    (db.auth_event.description.like('%Logged-in')))
+                ],
+                groupby=(db.auth_user.id, db.auth_event.description)
+            )
+
+        result = {}
+        for row in query:
+            result[row.auth_user.id] = row
+
+        for row in sset_count:
+            result[row.auth_user.id]['count'] = row.patient_count + row.run_count + row.generic_count
+
+        for row in logins:
+            result[row.auth_user.id]['last_login'] = row.last_login
+
+        return dict(message=T('group info'),
+                    result=result,
+                    group=group)
     return error_message(ACCESS_DENIED)
 
 

server/web2py/applications/vidjil/controllers/my_account.py

+# coding: utf8
+
+from datetime import datetime, timedelta
+import time
+import types
+
+if request.env.http_origin:
+    response.headers['Access-Control-Allow-Origin'] = request.env.http_origin
+    response.headers['Access-Control-Allow-Credentials'] = 'true'
+    response.headers['Access-Control-Max-Age'] = 86400
+
+ACCESS_DENIED = "access denied"
+
+def generic_get_group(element_type):
+    group = {}
+    separator = "|| ';' ||"
+    fields = ["config.name",
+              "%s_file.sample_set_id" % element_type,
+              "%s_file.config_id" % element_type,
+              "sample_set.sample_type"
+            ]
+    group_concat = "GROUP_CONCAT(DISTINCT  " + separator.join(fields)
+    group['patient'] =  group_concat + separator + "patient.first_name || ' ' || patient.last_name || '#')"
+    group['run'] = group_concat + separator + "run.name || '#')"
+    group['set'] = group_concat + separator + "generic.name || '#')"
+    return group
+
+def get_group_fuses():
+    return generic_get_group('fused')
+
+def get_group_analyses():
+    group = {}
+    separator = "|| ';' ||"
+    fields = ["analysis_file.sample_set_id",
+              "sample_set.sample_type"
+            ]
+    group_concat = "GROUP_CONCAT(DISTINCT  " + separator.join(fields)
+    group['patient'] =  group_concat + separator + "patient.first_name || ' ' || patient.last_name || '#')"
+    group['run'] = group_concat + separator + "run.name || '#')"
+    group['set'] = group_concat + separator + "generic.name || '#')"
+    return group
+
+def group_permissions():
+    return "GROUP_CONCAT(DISTINCT auth_permission.name)"
+
+def base_query(group_list):
+    return (db.auth_group.id.belongs(group_list) &
+        (db.auth_permission.group_id == db.auth_group.id))
+
+def access_query(group_list):
+    return (base_query(group_list) &
+            (db.auth_permission.table_name == 'sample_set') &
+            (db.auth_permission.name == 'access') &
+            (db.auth_permission.record_id > 0))
+
+def filter_by_tags(tags):
+        return ((db.tag.name.upper().belongs([t.upper() for t in tags])) &
+        (db.tag_ref.tag_id == db.tag.id) &
+        (db.tag_ref.table_name == 'sequence_file') &
+        (db.tag_ref.record_id == db.sample_set_membership.sequence_file_id)
+    )
+
+def base_left():
+    return [db.sample_set.on(
+                db.sample_set.id == db.auth_permission.record_id),
+            db.sample_set_membership.on(
+                db.sample_set_membership.sample_set_id == db.sample_set.id)]
+
+def get_permissions(group_list):
+    return db(db.auth_group.id.belongs(group_list)
+        ).select(
+            db.auth_group.role,
+            group_permissions(),
+            left=(db.auth_permission.on(
+                (db.auth_permission.group_id == db.auth_group.id) &
+                (db.auth_permission.table_name == 'sample_set') &
+                (db.auth_permission.record_id == 0))),
+            groupby=db.auth_group.role
+        )
+
+def get_most_used_tags(group_list):
+    left = [
+        db.sample_set_membership.on(
+            db.sample_set_membership.sample_set_id == db.auth_permission.record_id),
+        db.tag_ref.on(
+            (db.tag_ref.table_name == 'sequence_file') &
+            (db.tag_ref.record_id == db.sample_set_membership.sequence_file_id)),
+        db.tag.on(
+            db.tag_ref.tag_id == db.tag.id)
+    ]
+
+    return db(base_query(group_list)).select(
+        db.auth_group.role,
+        db.tag_ref.id.count().with_alias('count'),
+        db.tag.name,
+        left=left,
+        groupby=(db.auth_group.role, db.tag.name),
+        orderby=~db.tag_ref.id.count(),
+        limitby=(0,100)
+    )
+
+def index():
+    start = time.time()
+
+    since = datetime.today() - timedelta(days=30)
+
+    if auth.is_admin() and 'data' in request.vars:
+        import json
+        group_list = json.loads(request.vars['data'])['group_ids']
+    else:
+        group_list = [int(g.id) for g in auth.get_user_groups() + auth.get_user_group_parents()]
+
+    log.debug("group_list: %s" % group_list)
+
+    if "filter" not in request.vars :
+        request.vars["filter"] = ""
+
+    search, tags = parse_search(request.vars["filter"])
+
+    result = {}
+    perm_query = get_permissions(group_list)
+    for r in perm_query:
+        if(r.auth_group.role not in result):
+            result[r.auth_group.role] = {}
+            for set_type in ['patient', 'run', 'set']:
+                result[r.auth_group.role][set_type] = {'count': {'num_sets': 0, 'num_samples': 0, 'sample_type': 'generic' if set_type == 'set' else set_type}}
+                result[r.auth_group.role][set_type]['tags'] = []
+                result[r.auth_group.role][set_type]['statuses'] = ""
+                result[r.auth_group.role][set_type]['num_jobs'] = 0
+            result[r.auth_group.role]['fuses'] = []
+            result[r.auth_group.role]['analyses'] = []
+            result[r.auth_group.role]['tags'] = []
+        result[r.auth_group.role]['permissions'] = "" if r._extra[group_permissions()] is None else r._extra[group_permissions()]
+
+    query = access_query(group_list)
+    if (tags is not None and len(tags) > 0):
+        query = (query & filter_by_tags(tags))
+
+    group_statuses = "GROUP_CONCAT(DISTINCT scheduler_task.id || ';' || scheduler_task.status || '#')"
+    group_fuses = get_group_fuses()
+    group_analyses = get_group_analyses()
+
+    left = base_left() + [
+        db.sequence_file.on(
+            db.sequence_file.id == db.sample_set_membership.sequence_file_id),
+        db.results_file.on(
+            db.sample_set_membership.sequence_file_id == db.results_file.sequence_file_id),
+        db.scheduler_task.on(
+            (db.scheduler_task.id == db.results_file.scheduler_task_id) &
+            (db.scheduler_task.start_time >= since)),
+        db.fused_file.on(
+            (db.fused_file.sample_set_id == db.sample_set.id) &
+            (db.fused_file.fuse_date >= since)),
+        db.config.on(
+            db.config.id == db.fused_file.config_id),
+        db.analysis_file.on(
+            (db.analysis_file.sample_set_id == db.sample_set.id) &
+            (db.analysis_file.analyze_date > since))
+    ]
+
+    select = [
+        db.auth_group.id,
+        db.auth_group.role,
+        db.sample_set.sample_type.with_alias('sample_type'),
+        db.sample_set.id.count(distinct=True).with_alias('num_sets'),
+        db.sample_set_membership.sequence_file_id.count(distinct=True).with_alias('num_samples'),
+        group_statuses,
+    ]
+
+    queries = {}
+    for set_type in ['patient', 'run', 'generic']:
+        key = 'set' if set_type == 'generic' else set_type
+        set_query = (query &
+                    (db[set_type].sample_set_id == db.auth_permission.record_id))
+
+        queries[key] = db(set_query).select(
+            group_fuses[key],
+            group_analyses[key],
+            *select,
+            left=left,
+            groupby=(db.auth_group.role, db.sample_set.sample_type),
+            orderby=~db.scheduler_task.start_time
+        )
+
+    list_size = 50
+    for key in queries: # patient, run, set
+        query = queries[key]
+        for r in query: # 1 or 0 rows
+            result[r.auth_group.role][key]['count']['num_sets'] += r.num_sets
+            result[r.auth_group.role][key]['count']['num_samples'] += r.num_sets
+            result[r.auth_group.role][key]['count']['sample_type'] = r.sample_type
+            statuses = "" if r._extra[group_statuses] is None else "".join([s.strip('#').split(';')[1][0] for s in r._extra[group_statuses].strip(',').split(',') if s[-1] == '#'])
+            result[r.auth_group.role][key]['num_jobs'] = len(statuses)
+            result[r.auth_group.role][key]['statuses'] += statuses[:list_size]
+
+            fuses = [] if r._extra[group_fuses[key]] is None else [fuse.strip('#').split(';') for fuse in r._extra[group_fuses[key]].strip(',').split(',') if fuse[-1] == '#']
+            result[r.auth_group.role]['fuses'] += (fuses[:list_size])
+
+            analyses = [] if r._extra[group_analyses[key]] is None else [analysis.strip('#').split(';') for analysis in r._extra[group_analyses[key]].strip(',').split(',') if analysis[-1] == '#']
+            result[r.auth_group.role]['analyses'] += analyses[:list_size]
+
+    tags = get_most_used_tags(group_list) # list tags used without filtering
+    for r in tags:
+        if(r.tag.name is not None):
+            if r.count > 1:
+                tag_string = "%s (%d)" % (r.tag.name, r.count)
+            else:
+                tag_string = "%s" % r.tag.name
+            result[r.auth_group.role]['tags'].append(tag_string)