Attention une mise à jour du serveur va être effectuée le vendredi 16 avril entre 12h et 12h30. Cette mise à jour va générer une interruption du service de quelques minutes.

Commit 8c726f2a authored by HERBERT Ryan's avatar HERBERT Ryan

Reduce model overhead

Due to the amount of data being applied there was a considerable amount
of overhead involved in creating an object for each row in the query.

In order to attempt to preserve some of the generic code, the models
have been converted to helper classes.
parent e4a97f6b
...@@ -195,10 +195,9 @@ def all(): ...@@ -195,10 +195,9 @@ def all():
list.load_anon_permissions() list.load_anon_permissions()
result = list.get_values() result = list.get_values()
try: factory = ModelFactory()
headers = result[0].get_fields() helper = factory.get_instance(type=type)
except IndexError: fields = helper.get_fields()
headers = get_default_fields()
##sort result ##sort result
reverse = False reverse = False
...@@ -218,7 +217,7 @@ def all(): ...@@ -218,7 +217,7 @@ def all():
return dict(query = result, return dict(query = result,
headers = headers, fields = fields,
type = type, type = type,
isAdmin = isAdmin, isAdmin = isAdmin,
reverse = False) reverse = False)
......
class SampleSet(object): class SampleSet(object):
__slots__ = ('id', 'has_permission', 'info', 'creator', 'confs', 'conf_list', 'conf_id_list', 'most_used_conf', 'groups', 'group_list', 'file_count', 'size', 'sample_type', 'sequence_count', 'data_count')
def __init__(self, data):
self.id = data.id def __init__(self, type):
self.name = data.name self.type = type
self.has_permission = auth.can_modify_sample_set(self.id)
self.info = data.info
self.creator = data.creator
self.confs = ""
self.conf_list = []
self.conf_id_list = [-1]
self.most_used_conf = ""
self.groups = ""
self.group_list = []
self.file_count = 0
self.size = 0
self.sample_type = data.sample_type
self.sequence_count = None
self.data_count = None
def __getitem__(self, key): def __getitem__(self, key):
return getattr(self, key, None) return getattr(self, key, None)
def get_name(self): def get_name(self, data):
return 'sample set: %s' % self.name return 'sample set: %s' % data.name
def get_type(self): def get_info(self, data):
return self.sample_type return data.info
def get_info(self): def get_configs(self, data):
return self.info return data.conf_list
def get_configs(self): def get_config_urls(self, data):
return self.conf_list
def get_config_urls(self):
configs = [] configs = []
for conf in self.conf_list: for conf in data.conf_list:
filename = "(%s %s)" % (self.get_name(), conf['name']) filename = "(%s %s)" % (self.get_name(data), conf['name'])
if conf['fused_file'] is not None : if conf['fused_file'] is not None :
configs.append( configs.append(
str(A(conf['name'], str(A(conf['name'],
_href="index.html?sample_set=%d&config=%d" % (self.id, conf['id']), _type="text/html", _href="index.html?sample_set=%d&config=%d" % (data.id, conf['id']), _type="text/html",
_onclick="event.preventDefault();event.stopPropagation();if( event.which == 2 ) { window.open(this.href); } else { db.load_data( { 'sample_set' : '%d', 'config' : %d }, '%s' ); }" % (self.id, conf['id'], filename)))) _onclick="event.preventDefault();event.stopPropagation();if( event.which == 2 ) { window.open(this.href); } else { db.load_data( { 'sample_set' : '%d', 'config' : %d }, '%s' ); }" % (data.id, conf['id'], filename))))
else: else:
configs.append(conf['name']) configs.append(conf['name'])
return XML(", ".join(configs)) return XML(", ".join(configs))
def get_groups(self): def get_groups(self, data):
return self.group_list return data.group_list
def get_groups_string(self): def get_groups_string(self, data):
return ', '.join([group for group in self.group_list if group != 'admin']) return ', '.join([group for group in data.group_list if group != 'admin'])
def get_creator(self): def get_creator(self, data):
return self.creator return data.creator
def get_files(self): def get_files(self, data):
return '%d (%s)' % (self.file_count, vidjil_utils.format_size(self.size)) return '%d (%s)' % (data.file_count, vidjil_utils.format_size(data.size))
def get_fields(self): def get_fields(self):
fields = [] fields = []
fields.append({'name': 'name', 'sort': 'name', 'call': self.get_name, 'width': 200, 'public': True}) fields.append({'name': 'name', 'sort': 'name', 'call': self.get_name, 'width': 200, 'public': True})
fields.append({'name': 'info', 'sort': 'info', 'call': self.get_info, 'width': None, 'public': True}) fields.append({'name': 'info', 'sort': 'info', 'call': self.get_info, 'width': None, 'public': True})
fields.append({'name': 'results', 'sort': 'confs', 'call': self.get_config_urls, 'width': None, 'public': True}) fields.append({'name': 'results', 'sort': 'confs', 'call': self.get_config_urls, 'width': None, 'public': True})
fields.append({'name': 'groups', 'sort': 'groups', 'call': self.get_groups_string, 'width': 100, 'public': False}) if auth.is_admin():
fields.append({'name': 'creator', 'sort': 'creator', 'call': self.get_creator, 'width': 100, 'public': False}) fields.append({'name': 'groups', 'sort': 'groups', 'call': self.get_groups_string, 'width': 100, 'public': False})
fields.append({'name': 'creator', 'sort': 'creator', 'call': self.get_creator, 'width': 100, 'public': False})
fields.append({'name': 'files', 'sort': 'file_count', 'call': self.get_files, 'width': 100, 'public': True}) fields.append({'name': 'files', 'sort': 'file_count', 'call': self.get_files, 'width': 100, 'public': True})
return fields return fields
def get_sequence_count(self): def get_sequence_count(self, data):
if self.sequence_count is None: if data.sequence_count is None:
self.sequence_count = db( (db.sequence_file.id == db.sample_set_membership.sequence_file_id) data.sequence_count = db( (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
&(db.sample_set_membership.sample_set_id == db.patient.sample_set_id) &(db.sample_set_membership.sample_set_id == db[self.type].sample_set_id)
&(db.patient.id == self.id)).count() &(db[self.type].id == data.id)).count()
return self.sequence_count return data.sequence_count
def get_data_count(self): def get_data_count(selfi, data):
if self.data_count is None: if data.data_count is None:
self.data_count = db( (db.sequence_file.id == db.sample_set_membership.sequence_file_id) data.data_count = db( (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
&(db.sample_set_membership.sample_set_id == db.patient.sample_set_id) &(db.sample_set_membership.sample_set_id == db[self.type].sample_set_id)
&(db.patient.id == self.id) &(db[self.type].id == data.id)
&(db.results_file.sequence_file_id == db.sequence_file.id)).count() &(db.results_file.sequence_file_id == db.sequence_file.id)).count()
return self.data_count return data.data_count
def get_sample_set(id):
data = db.sample_set[id]
return SampleSet(data)
def get_default_fields():
fields = []
fields.append({'name': 'name', 'sort': 'name', 'width': 200, 'public': True})
fields.append({'name': 'info', 'sort': 'info', 'width': None, 'public': True})
fields.append({'name': 'results', 'sort': 'configs', 'width': None, 'public': True})
fields.append({'name': 'groups', 'sort': 'groups', 'width': 100, 'public': False})
fields.append({'name': 'creator', 'sort': 'creator', 'width': 100, 'public': False})
fields.append({'name': 'files', 'sort': 'files', 'width': 100, 'public': True})
return fields
class Patient(SampleSet): class Patient(SampleSet):
__slots__ = ('id', 'first_name', 'has_permission', 'info', 'creator', 'confs', 'conf_list', 'conf_id_list', 'most_used_conf', 'groups', 'group_list', 'file_count', 'size', 'sample_type', 'sequence_count', 'data_count', 'last_name', 'birth', 'anon_allowed')
def __init__(self, data): def __init__(self, type):
super(Patient, self).__init__(data) super(Patient, self).__init__(type)
self.last_name = data.last_name
self.birth = data.birth
#TODO
self.anon_allowed = True
def get_fields(self): def get_fields(self):
fields = super(Patient, self).get_fields() fields = super(Patient, self).get_fields()
fields[0] = {'name': 'name', 'sort': 'last_name', 'call': self.get_name, 'width': 100, 'public': True}
fields.insert(1, {'name': 'birth', 'sort': 'birth', 'call': self.get_birth, 'width': 100, 'public': True}) fields.insert(1, {'name': 'birth', 'sort': 'birth', 'call': self.get_birth, 'width': 100, 'public': True})
print fields
return fields return fields
def get_name(self): def get_name(self, data):
return vidjil_utils.anon_names(self.id, self.name, self.last_name, self.anon_allowed) return vidjil_utils.anon_names(data.id, data.first_name, data.last_name, data.anon_allowed)
def get_birth(self): def get_birth(self, data):
return self.birth return "%s" % str(data.birth) if data.birth is not None else ''
def get_patient(id):
data = db.sample_set[id]
return Patient(data)
class Run(SampleSet): class Run(SampleSet):
def __init__(self, data): def __init__(self, type):
super(Run, self).__init__(data) super(Run, self).__init__(type)
self.run_date = data.date
self.id_label = data.id_label
self.sequencer = data.sequencer
self.pcr = data.pcr
def get_fields(self): def get_fields(self):
fields = super(Run, self).get_fields() fields = super(Run, self).get_fields()
fields.insert(1, {'name': 'run_date', 'sort': 'run_date', 'call': self.get_run_date, 'width': 100, 'public': True}) fields.insert(1, {'name': 'run_date', 'sort': 'run_date', 'call': self.get_run_date, 'width': 100, 'public': True})
return fields return fields
def get_name(self): def get_name(self, data):
return self.name return data.name
def get_embellished_name(self): def get_embellished_name(self, data):
return 'run: %s' % self.name return 'run: %s' % data.name
def get_birth(self): def get_run_date(self, data):
return self.birth return data.run_date
def get_run(id):
data = db.run[id]
return Run(data)
class SampleSetList(): class SampleSetList():
def __init__(self, type): def __init__(self, type):
print type self.type = type
query_gss = db( query_gss = db(
(auth.vidjil_accessible_query(PermissionEnum.read.value, db.sample_set)) & (auth.vidjil_accessible_query(PermissionEnum.read.value, db[type]))
(db.sample_set.sample_type == type)
).select( ).select(
db.sample_set.ALL, db[type].ALL,
orderby = ~db.sample_set.id orderby = ~db[type].id
) )
auth.load_permissions(PermissionEnum.admin.value, 'sample_set') auth.load_permissions(PermissionEnum.admin.value, type)
self.sample_sets = {} auth.load_permissions(PermissionEnum.anon.value, type)
factory = ModelFactory() self.elements = {}
for row in query_gss: for row in query_gss:
self.sample_sets[row.id] = factory.get_instance(type, data=row) self.elements[row.id] = row
self.sample_set_ids = self.sample_sets.keys() self.elements[row.id].file_count = 0
self.elements[row.id].size = 0
self.elements[row.id].conf_list = []
self.elements[row.id].conf_id_list = [-1]
self.elements[row.id].most_used_conf = ""
self.elements[row.id].groups = ""
self.elements[row.id].group_list = []
self.elements[row.id].has_permission = auth.can_modify(type, row.id)
self.elements[row.id].anon_allowed = auth.can_view_info(type, row.id)
self.element_ids = self.elements.keys()
def load_creator_names(self): def load_creator_names(self):
query_creator = db( query_creator = db(
(db.sample_set.creator == db.auth_user.id) (db[self.type].creator == db.auth_user.id)
& (db.sample_set.id.belongs(self.sample_set_ids)) & (db[self.type].id.belongs(self.element_ids))
).select( ).select(
db.sample_set.id, db.auth_user.last_name db[self.type].id, db.auth_user.last_name
) )
for i, row in enumerate(query_creator) : for i, row in enumerate(query_creator) :
self.sample_sets[row.sample_set.id].creator = row.auth_user.last_name self.elements[row[self.type].id].creator = row.auth_user.last_name
def load_sample_information(self): def load_sample_information(self):
query_sample = db( query_sample = db(
(db.sample_set_membership.sample_set_id == db.sample_set.id) (db.sample_set_membership.sample_set_id == db[self.type].sample_set_id)
&(db.sequence_file.id == db.sample_set_membership.sequence_file_id) &(db.sequence_file.id == db.sample_set_membership.sequence_file_id)
&(db.sample_set.id.belongs(self.sample_set_ids)) &(db[self.type].id.belongs(self.element_ids))
).select( ).select(
db.sample_set.id, db.sequence_file.size_file db[self.type].id, db.sequence_file.size_file
) )
for i, row in enumerate(query_sample) : for i, row in enumerate(query_sample) :
self.sample_sets[row.sample_set.id].file_count += 1 self.elements[row[self.type].id].file_count += 1
self.sample_sets[row.sample_set.id].size += row.sequence_file.size_file self.elements[row[self.type].id].size += row.sequence_file.size_file
def load_config_information(self): def load_config_information(self):
query = db( query = db(
(db.sample_set.id == db.fused_file.sample_set_id) & (db[self.type].sample_set_id == db.fused_file.sample_set_id) &
(db.fused_file.config_id == db.config.id) & (db.fused_file.config_id == db.config.id) &
(auth.vidjil_accessible_query(PermissionEnum.read_config.value, db.config) | (auth.vidjil_accessible_query(PermissionEnum.read_config.value, db.config) |
auth.vidjil_accessible_query(PermissionEnum.admin_config.value, db.config) ) & auth.vidjil_accessible_query(PermissionEnum.admin_config.value, db.config) ) &
(db.sample_set.id.belongs(self.sample_set_ids)) (db[self.type].id.belongs(self.element_ids))
).select( ).select(
db.sample_set.id, db.config.name, db.config.id, db.fused_file.fused_file db[self.type].id, db.config.name, db.config.id, db.fused_file.fused_file
) )
for i, row in enumerate(query) : for i, row in enumerate(query) :
self.sample_sets[row.sample_set.id].conf_list.append( self.elements[row[self.type].id].conf_list.append(
{'id': row.config.id, 'name': row.config.name, 'fused_file': row.fused_file.fused_file}) {'id': row.config.id, 'name': row.config.name, 'fused_file': row.fused_file.fused_file})
self.sample_sets[row.sample_set.id].conf_id_list.append(row.config.id) self.elements[row[self.type].id].conf_id_list.append(row.config.id)
for key, row in self.sample_sets.iteritems(): for key, row in self.elements.iteritems():
row.most_used_conf = max(set(row.conf_id_list), key=row.conf_id_list.count) row.most_used_conf = max(set(row.conf_id_list), key=row.conf_id_list.count)
def load_permitted_groups(self): def load_permitted_groups(self):
query = db( query = db(
((db.sample_set.id == db.auth_permission.record_id) | (db.auth_permission.record_id == 0)) & ((db[self.type].id == db.auth_permission.record_id) | (db.auth_permission.record_id == 0)) &
(db.auth_permission.table_name == 'sample_set') & (db.auth_permission.table_name == self.type) &
(db.auth_permission.name == PermissionEnum.read.value) & (db.auth_permission.name == PermissionEnum.access.value) &
(db.auth_group.id == db.auth_permission.group_id) & (db.auth_group.id == db.auth_permission.group_id)
(db.sample_set.id.belongs(self.sample_set_ids)) & (db[self.type].id.belongs(self.element_ids))
).select( ).select(
db.sample_set.id, db.auth_group.role db[self.type].id, db.auth_group.role
) )
for i, row in enumerate(query) : for i, row in enumerate(query) :
self.sample_sets[row.sample_set.id].group_list.append(row.auth_group.role.replace('user_','u')) self.elements[row[self.type].id].group_list.append(row.auth_group.role.replace('user_','u'))
for key, row in self.sample_sets.iteritems(): for key, row in self.elements.iteritems():
row.groups = ", ".join(filter(lambda g: g != 'admin', set(row.group_list))) row.groups = ", ".join(filter(lambda g: g != 'admin', set(row.group_list)))
def load_anon_permissions(self): def load_anon_permissions(self):
query = db( query = db(
(db.auth_permission.name == "anon") & (db.auth_permission.name == "anon") &
(db.auth_permission.table_name == "sample_set") & (db.auth_permission.table_name == self.type) &
(db.sample_set.id == db.auth_permission.record_id ) & (db[self.type].id == db.auth_permission.record_id ) &
(db.auth_group.id == db.auth_permission.group_id ) & (db.auth_group.id == db.auth_permission.group_id ) &
(db.auth_membership.user_id == auth.user_id) & (db.auth_membership.user_id == auth.user_id) &
(db.auth_membership.group_id == db.auth_group.id) & (db.auth_membership.group_id == db.auth_group.id) &
(db.sample_set.id.belongs(self.sample_set_ids)) (db[self.type].id.belongs(self.element_ids))
).select( ).select(
db.sample_set.id db[self.type].id
) )
for i, row in enumerate(query) : for i, row in enumerate(query) :
self.sample_sets[row.id].anon_allowed = True self.elements[row.id].anon_allowed = True
def get_values(self): def get_values(self):
return self.sample_sets.values() return self.elements.values()
...@@ -11,5 +11,5 @@ class ModelFactory(): ...@@ -11,5 +11,5 @@ class ModelFactory():
#filler constructor, do we need a class here ? #filler constructor, do we need a class here ?
self.id = 0 self.id = 0
def get_instance(self, type, **kwargs): def get_instance(self, type):
return FactoryEnum[type].value(**kwargs) return FactoryEnum[type].value(type)
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
<table class="db_table" id="table"> <table class="db_table" id="table">
<thead> <thead>
<tr> <tr>
{{ for header in headers: }} {{ for header in fields: }}
{{if isAdmin or header['public']:}} {{if isAdmin or header['public']:}}
<td class="{{if header['width']:}}column_{{=header['width']}}{{pass}} pointer" <td class="{{if header['width']:}}column_{{=header['width']}}{{pass}} pointer"
onclick="db.call('sample_set/all', {'sort' : '{{=header['sort']}}', 'filter' : '{{=request.vars["filter"]}}' onclick="db.call('sample_set/all', {'sort' : '{{=header['sort']}}', 'filter' : '{{=request.vars["filter"]}}'
...@@ -34,10 +34,10 @@ ...@@ -34,10 +34,10 @@
{{for data in query :}} {{for data in query :}}
<tr class="pointer" onclick="db.call('sample_set/index', {'id' :'{{=data.id}}' , 'config_id' : {{=data.most_used_conf}} } )" > <tr class="pointer" onclick="db.call('sample_set/index', {'id' :'{{=data.sample_set_id}}' , 'config_id' : {{=data.most_used_conf}} } )" >
{{ for field in data.get_fields(): }} {{ for field in fields: }}
<td> <td>
{{= field['call']()}} {{= field['call'](data)}}
</td> </td>
{{ pass }} {{ pass }}
{{ total_file_count += data.file_count ; total_size += data.size }} {{ total_file_count += data.file_count ; total_size += data.size }}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment