Commit 172ac2d3 authored by Mikaël Salson's avatar Mikaël Salson

Merge branch 'feature-s/single-query-tests' into 'dev'

Single query, improve set queries

Closes #3169

See merge request !832
parents e7cca318 8191cc98
Pipeline #188721 passed with stages
in 11 minutes and 4 seconds
......@@ -61,7 +61,29 @@ def index():
if request.vars["config_id"] and request.vars["config_id"] != "-1" and request.vars["config_id"] != "None":
config_id = long(request.vars["config_id"])
config_name = db.config[request.vars["config_id"]].name
config = True
elif request.vars["config_id"] and request.vars["config_id"] == "-1":
most_used_query = db(
(db.fused_file.sample_set_id == sample_set.id)
).select(
db.fused_file.config_id.with_alias('id'),
db.fused_file.id.count().with_alias('use_count'),
groupby=db.fused_file.config_id,
orderby=db.fused_file.id.count(),
limitby=(0,1)
)
if len(most_used_query) > 0:
config_id = most_used_query[0].id
config = True
else:
config_id = -1
config = False
else:
config_id = -1
config = False
if config :
config_name = db.config[config_id].name
fused = db(
(db.fused_file.sample_set_id == sample_set_id)
......@@ -73,7 +95,6 @@ def index():
).select(orderby=~db.analysis_file.analyze_date)
config = True
fused_count = fused.count()
fused_file = fused.select()
fused_filename = info_file["filename"] +"_"+ config_name + ".vidjil"
......@@ -81,17 +102,6 @@ def index():
analysis_file = analysis
analysis_filename = info_file["filename"]+"_"+ config_name + ".analysis"
else:
config_id = -1
config = False
fused_count = 0
fused_file = ""
fused_filename = ""
analysis_count = 0
analysis_file = ""
analysis_filename = ""
if config :
query =[]
query2 = db(
......@@ -113,6 +123,12 @@ def index():
previous=row.sequence_file.id
else:
fused_count = 0
fused_file = ""
fused_filename = ""
analysis_count = 0
analysis_file = ""
analysis_filename = ""
query = db(
(db.sequence_file.id == db.sample_set_membership.sequence_file_id)
......@@ -194,8 +210,7 @@ def all():
step = None
page = None
is_not_filtered = "sort" not in request.vars and "filter" not in request.vars
if request.vars['page'] is not None and is_not_filtered:
if request.vars['page'] is not None:
page = int(request.vars['page'])
step = 50
......@@ -206,21 +221,27 @@ def all():
search, tags = parse_search(request.vars["filter"])
group_ids = get_involved_groups()
list = SampleSetList(type, page, step, tags=tags)
list.load_creator_names()
list.load_sample_information()
list.load_config_information()
if isAdmin or len(get_group_list(auth)) > 1:
list.load_permitted_groups()
list.load_anon_permissions()
result = list.get_values()
factory = ModelFactory()
helper = factory.get_instance(type=type)
f = time.time()
slist = SampleSetList(helper, page, step, tags, search)
log.debug("list loaded (%.3fs)" % (time.time() - f))
mid = time.time()
set_ids = set([s.sample_set_id for s in slist.result])
admin_permissions = [s.id for s in db(auth.vidjil_accessible_query(PermissionEnum.admin.value, db.sample_set)).select(db.sample_set.id)]
admin_permissions = list(set(admin_permissions) & set_ids)
log.debug("permission load (%.3fs)" % (time.time() - mid))
# failsafe if filtered display all results
step = len(list) if step is None else step
step = len(slist) if step is None else step
page = 0 if page is None else page
result = slist.result
factory = ModelFactory()
helper = factory.get_instance(type=type)
fields = helper.get_fields()
sort_fields = helper.get_sort_fields()
......@@ -233,7 +254,6 @@ def all():
else:
result = sorted(result, key = lambda row : row.id, reverse=not reverse)
result = helper.filter(search, result)
log.info("%s list %s" % (request.vars["type"], search), extra={'user_id': auth.user.id,
'record_id': None,
'table_name': "sample_set"})
......@@ -244,6 +264,7 @@ def all():
fields = fields,
helper = helper,
group_ids = group_ids,
admin_permissions = admin_permissions,
isAdmin = isAdmin,
reverse = reverse,
step = step,
......@@ -270,13 +291,12 @@ def stats():
search, tags = parse_search(request.vars["filter"])
group_ids = get_involved_groups()
list = SampleSetList(type, tags=tags)
list.load_sample_information()
list.load_anon_permissions()
result = list.get_values()
factory = ModelFactory()
helper = factory.get_instance(type=type)
slist = SampleSetList(helper, tags=tags)
result = slist.result
fields = helper.get_reduced_fields()
##sort result
......@@ -294,7 +314,7 @@ def stats():
log.info("%s stat list %s" % (request.vars["type"], search), extra={'user_id': auth.user.id,
'record_id': None,
'table_name': "sample_set"})
log.debug("stat list (%.3fs)" % time.time()-start)
log.debug("stat list (%.3f s)" % (time.time()-start))
return dict(query = result,
fields = fields,
......
......@@ -35,7 +35,8 @@ class SampleSet(object):
return self.tag_decorator.sanitize(text)
def get_configs(self, data):
return data['conf_list']
conf_list = get_conf_list_select()
return data._extra[conf_list]
def get_list_path(self):
return '/sample_set/all'
......@@ -48,28 +49,41 @@ class SampleSet(object):
http_origin = ""
if request.env['HTTP_ORIGIN'] is not None:
http_origin = request.env['HTTP_ORIGIN'] + "/"
for conf in data['conf_list']:
filename = "(%s %s)" % (self.get_name(data), conf['name'])
if conf['fused_file'] is not None :
key = get_conf_list_select()
conf_list = [] if data._extra[key] is None else data._extra[key].split(',')
for conf in conf_list:
c = conf.split(';')
filename = "(%s %s)" % (self.get_name(data), c[1])
if c[2] is not None :
configs.append(
str(A(conf['name'],
_href=http_origin + "?sample_set_id=%d&config=%d" % (data['sample_set_id'], conf['id']), _type="text/html",
_onclick="event.preventDefault();event.stopPropagation();if( event.which == 2 ) { window.open(this.href); } else { myUrl.loadUrl(db, { 'sample_set_id' : '%d', 'config' : %d }, '%s' ); }" % (data['sample_set_id'], conf['id'], filename))))
str(A(c[1],
_href="index.html?sample_set_id=%d&config=%s" % (data['sample_set_id'], c[0]), _type="text/html",
_onclick="event.preventDefault();event.stopPropagation();if( event.which == 2 ) { window.open(this.href); } else { myUrl.loadUrl(db, { 'sample_set_id' : '%d', 'config' : %s }, '%s' ); }" % (data['sample_set_id'], c[0], filename))))
else:
configs.append(conf['name'])
configs.append(c[1])
return XML(", ".join(configs))
def get_groups(self, data):
return data['group_list']
key = get_group_names_select()
return data._extra[key]
def get_groups_string(self, data):
return ', '.join([group for group in data['group_list'] if group != 'admin'])
key = get_group_names_select()
group_list = [] if data._extra[key] is None else data._extra[key].split(',')
return ', '.join([group for group in group_list if group != 'admin'])
def get_creator(self, data):
return data['creator']
def get_files_values(self, data):
key = get_file_sizes_select()
size = 0 if data._extra[key] is None else data._extra[key]
file_count = data.file_count
return file_count, size
def get_files(self, data):
return '%d (%s)' % (data['file_count'], vidjil_utils.format_size(data['size']))
file_count, size = self.get_files_values(data)
return '%d (%s)' % (file_count, vidjil_utils.format_size(size))
def get_fields(self):
fields = []
......@@ -149,6 +163,18 @@ class SampleSet(object):
def validate(self, data):
pass
@abstractmethod
def get_dedicated_fields(self):
pass
@abstractmethod
def get_dedicated_group(self):
pass
@abstractmethod
def get_filtered_fields(self, search):
pass
def get_sample_name(sample_set_id):
'''
Return the name associated with a sample set (eg. a run or a
......@@ -189,3 +215,17 @@ def get_sample_set_id_from_results_file(results_file_id):
).select(db.sample_set_membership.sample_set_id).first().sample_set_id
return sample_set_id
def get_conf_list_select():
return "GROUP_CONCAT(DISTINCT (config.id || ';' || config.name || ';' || fused_file.fused_file))"
def get_config_ids_select():
return "GROUP_CONCAT(DISTINCT config.id)"
def get_config_names_select():
return "GROUP_CONCAT(DISTINCT config.name)"
def get_group_names_select():
return "GROUP_CONCAT(DISTINCT auth_group.role)"
def get_file_sizes_select():
return "COUNT(DISTINCT sequence_file.id) * SUM(sequence_file.size_file) / COUNT(*)"
......@@ -109,6 +109,7 @@ auth.messages.group_description = 'Group of user %(id)04d - %(first_name)s %(las
from gluon.contrib.login_methods.rpx_account import use_janrain
use_janrain(auth, filename='private/janrain.key')
# TODO: create a custom adapter ?
if defs.DB_ADDRESS.split(':')[0] == 'mysql':
db.executesql("SET sql_mode='PIPES_AS_CONCAT,NO_BACKSLASH_ESCAPES';")
......@@ -288,11 +289,14 @@ db.define_table('tag_ref',
Field('table_name', 'string'),
Field('record_id', 'integer'))
# try to create an index on these un-indexed columns, if it fails, we assume they already exist
try:
db.executesql('CREATE INDEX table_name_index ON tag_ref (table_name);')
db.executesql('CREATE INDEX record_id_index ON tag_ref (record_id);')
db.executesql('CREATE INDEX name_index ON auth_permission (name);')
db.executesql('CREATE INDEX record_id_index ON auth_permission (record_id);')
except:
pass
pass
## after defining tables, uncomment below to enable auditing
auth.enable_record_versioning(db)
......
......@@ -32,3 +32,17 @@ class Generic(SampleSet):
if data["name"].find("|") >= 0:
error.append("illegal character '|' in name")
return error
def get_dedicated_fields(self):
table = db[self.type]
return [
table.name.with_alias('name')
]
def get_dedicated_group(self):
table = db[self.type]
return [table.name]
def get_filtered_fields(self, search):
table = db[self.type]
return table.name.contains(search)
......@@ -63,6 +63,23 @@ class Patient(SampleSet):
error.append("illegal character '|' in name")
return error
def get_dedicated_fields(self):
table = db[self.type]
return [
table.first_name.with_alias('first_name'),
table.last_name.with_alias('last_name'),
table.birth.with_alias('birth'),
]
def get_dedicated_group(self):
table = db[self.type]
return [table.first_name, table.last_name, table.birth]
def get_filtered_fields(self, search):
table = db[self.type]
return (table.first_name.contains(search) |
table.last_name.contains(search))
def get_name_filter_query(self, query):
if query is None or query == '':
return (db[self.type].id > 0)
......
......@@ -61,3 +61,18 @@ class Run(SampleSet):
error.append("illegal character '|' in name")
return error
def get_dedicated_fields(self):
table = db[self.type]
return [
table.name.with_alias('name'),
table.run_date.with_alias('run_date'),
]
def get_dedicated_group(self):
table = db[self.type]
return [table.name]
def get_filtered_fields(self, search):
table = db[self.type]
return table.name.contains(search)
......@@ -4,15 +4,65 @@ class SampleSetList():
This class is used to load all the required information for such a list.
'''
def __init__(self, type, page=None, step=None, tags=None):
self.type = type
def __init__(self, helper, page=None, step=None, tags=None, search=None):
self.type = helper.get_type()
s_table = db[self.type]
limitby = None
if page is not None and step is not None:
limitby = (page*step, (page+1)*step+1) # one more element to indicate if another page exists
query = ((auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set)) &
(db[type].sample_set_id == db.sample_set.id))
left = [db.sample_set_membership.on(s_table.sample_set_id == db.sample_set_membership.sample_set_id),
db.sequence_file.on(db.sample_set_membership.sequence_file_id == db.sequence_file.id),
db.fused_file.on(s_table.sample_set_id == db.fused_file.sample_set_id),
db.config.on(db.fused_file.config_id == db.config.id),
db.auth_permission.with_alias('generic_perm').on(
(db.generic_perm.table_name == 'sample_set') &
(db.generic_perm.record_id == s_table.sample_set_id) &
(db.generic_perm.name == PermissionEnum.access.value)),
db.auth_group.on(
(db.generic_perm.group_id == db.auth_group.id)),
db.auth_membership.on(db.auth_group.id == db.auth_membership.group_id),
db.auth_user.on(
db.auth_user.id == s_table.creator)
]
# SQL string with GROUP_CONCAT queries
group_configs = get_conf_list_select()
group_config_ids = get_config_ids_select()
group_config_names = get_config_names_select()
group_group_names = get_group_names_select()
group_file_sizes = get_file_sizes_select()
dedicated_fields = helper.get_dedicated_fields()
groupby = [s_table.id, s_table.sample_set_id, s_table.info, db.auth_user.last_name]
groupby += helper.get_dedicated_group()
join = [s_table.on(s_table.sample_set_id == db.sample_set.id)]
query = (auth.vidjil_accessible_query('read', db.sample_set))
if search is not None and search != "":
query = (query &
(helper.get_filtered_fields(search) |
s_table.info.contains(search) |
db.config.name.contains(search) |
db.auth_group.role.contains(search) |
db.auth_user.last_name.contains(search)))
select = [
s_table.id.with_alias('id'),
s_table.sample_set_id.with_alias('sample_set_id'),
s_table.info.with_alias('info'),
db.auth_user.last_name.with_alias('creator'),
db.sequence_file.id.count(True).with_alias('file_count'),
group_file_sizes,
group_configs,
group_config_ids,
group_config_names,
group_group_names
]
if (tags is not None and len(tags) > 0):
query = filter_by_tags(query, self.type, tags)
......@@ -20,127 +70,35 @@ class SampleSetList():
query_gss = db(
query
).select(
db[type].ALL,
db.tag_ref.record_id,
db.tag_ref.table_name,
count,
*select + dedicated_fields,
join=join,
left=left,
limitby = limitby,
orderby = ~db[type].id,
groupby = db.tag_ref.record_id|db.tag_ref.table_name,
having = count >= len(tags)
orderby = ~db[self.type].id,
groupby = groupby + [db.tag_ref.record_id|db.tag_ref.table_name],
having = count >= len(tags),
cacheable=True
)
else:
query_gss = db(
query
).select(
db[type].ALL,
*select + dedicated_fields,
join=join,
left=left,
limitby = limitby,
orderby = ~db[type].id
groupby = groupby,
orderby = ~db[self.type].id,
cacheable=True
)
step = len(query_gss) if step is None else step
auth.load_permissions(PermissionEnum.admin.value, type)
auth.load_permissions(PermissionEnum.anon.value, type)
self.elements = {}
for row in query_gss:
if (tags is not None and len(tags) > 0):
key = row[type]
else:
key = row
self.elements[key.id] = key
self.elements[key.id].file_count = 0
self.elements[key.id].size = 0
self.elements[key.id].conf_list = []
self.elements[key.id].conf_id_list = [-1]
self.elements[key.id].most_used_conf = ""
self.elements[key.id].groups = ""
self.elements[key.id].group_list = []
#self.elements[key.id].has_permission = auth.can_modify_sample_set(key.sample_set_id)
self.elements[key.id].has_permission = auth.can_modify_subset(type, key.id)
self.elements[key.id].anon_allowed = auth.can_view_info(type, key.id)
self.element_ids = self.elements.keys()
def load_creator_names(self):
query_creator = db(
(db[self.type].creator == db.auth_user.id)
& (db[self.type].id.belongs(self.element_ids))
).select(
db[self.type].id, db.auth_user.last_name
)
for i, row in enumerate(query_creator) :
self.elements[row[self.type].id].creator = row.auth_user.last_name
def load_sample_information(self):
query_sample = db(
(db.sample_set_membership.sample_set_id == db[self.type].sample_set_id)
&(db.sequence_file.id == db.sample_set_membership.sequence_file_id)
&(db[self.type].id.belongs(self.element_ids))
).select(
db[self.type].id, db.sequence_file.size_file
)
for i, row in enumerate(query_sample) :
self.elements[row[self.type].id].file_count += 1
self.elements[row[self.type].id].size += row.sequence_file.size_file
def load_config_information(self):
query = db(
(db[self.type].sample_set_id == db.fused_file.sample_set_id) &
(db.fused_file.config_id == db.config.id) &
(auth.vidjil_accessible_query(PermissionEnum.read_config.value, db.config) |
auth.vidjil_accessible_query(PermissionEnum.admin_config.value, db.config) ) &
(db[self.type].id.belongs(self.element_ids))
).select(
db[self.type].id, db.config.name, db.config.id, db.fused_file.fused_file
)
for i, row in enumerate(query) :
self.elements[row[self.type].id].conf_list.append(
{'id': row.config.id, 'name': row.config.name, 'fused_file': row.fused_file.fused_file})
self.elements[row[self.type].id].conf_id_list.append(row.config.id)
for key, row in self.elements.iteritems():
row.most_used_conf = max(set(row.conf_id_list), key=row.conf_id_list.count)
row.confs = ", ".join(list(set(map(lambda elem: elem['name'], row.conf_list))))
def load_permitted_groups(self):
query = db(
((db[self.type].id == db.auth_permission.record_id) | (db.auth_permission.record_id == 0)) &
(db.auth_permission.table_name == self.type) &
(db.auth_permission.name == PermissionEnum.access.value) &
(db.auth_group.id == db.auth_permission.group_id)
& (db[self.type].id.belongs(self.element_ids))
).select(
db[self.type].id, db.auth_group.role, db.auth_group.id
)
for i, row in enumerate(query) :
self.elements[row[self.type].id].group_list.append(row.auth_group.role.replace('user_','u'))
for key, row in self.elements.iteritems():
row.groups = ", ".join(filter(lambda g: g != 'admin', set(row.group_list)))
def load_anon_permissions(self):
query = db(
(db.auth_permission.name == "anon") &
(db.auth_permission.table_name == self.type) &
(db[self.type].id == db.auth_permission.record_id ) &
(db.auth_group.id == db.auth_permission.group_id ) &
(db.auth_membership.user_id == auth.user_id) &
(db.auth_membership.group_id == db.auth_group.id) &
(db[self.type].id.belongs(self.element_ids))
).select(
db[self.type].id
)
for i, row in enumerate(query) :
self.elements[row.id].anon_allowed = True
def get_values(self):
return self.elements.values()
self.result = query_gss
def __len__(self):
return len(self.element_ids)
return len(self.result)
def filter_by_tags(query, table, tags):
if tags is not None and len(tags) > 0:
......
......@@ -44,8 +44,8 @@ class DBInitialiser(object):
def get_set_dict(self, set_type, sample_set_id, i):
if set_type == defs.SET_TYPE_PATIENT:
return dict(id_label="", first_name="patient", last_name=i, birth="2010-10-10", info="test patient %d #test%d" % (i, i), sample_set_id=sample_set_id)
d = dict(name="%s %d" % (set_type, i), info="test %s %d #test%d" % (set_type, i, i), sample_set_id=sample_set_id)
return dict(id_label="", first_name="patient", last_name=i, birth="2010-10-10", info="test patient %d #test%d" % (i, i), sample_set_id=sample_set_id, creator=1)
d = dict(name="%s %d" % (set_type, i), info="test %s %d #test%d" % (set_type, i, i), sample_set_id=sample_set_id, creator=1)
if set_type == defs.SET_TYPE_RUN:
d['id_label'] = ""
return d
......
......@@ -16,32 +16,39 @@ class SamplesetlistModel(unittest.TestCase):
auth.login_bare("test@vidjil.org", "123456")
def testInit(self):
slist = SampleSetList('patient')
self.assertTrue(len(slist.element_ids) > 0, "The sample set list was not expected to be empty")
factory = ModelFactory()
helper = factory.get_instance(type='patient')
slist = SampleSetList(helper)
self.assertTrue(len(slist) > 0, "The sample set list was not expected to be empty")
def testCreatorNames(self):
slist = SampleSetList('patient')
slist.load_creator_names()
values = slist.get_values()
factory = ModelFactory()
helper = factory.get_instance(type='patient')
slist = SampleSetList(helper)
values = slist.result
first = values[0]
name = first.creator
name = helper.get_creator(first)
self.assertFalse(name == "", "load_creator_names failed to retrieve a username")