Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vidjil
vidjil
Commits
3041dbc5
Commit
3041dbc5
authored
Nov 07, 2018
by
Mathieu Giraud
Browse files
Merge branch 'feature-sc/3506-speedup-clonedb' into 'dev'
Speedup clonedb Closes
#3506
See merge request
!316
parents
705baa45
67ca46d4
Pipeline
#48391
passed with stages
in 6 minutes and 51 seconds
Changes
9
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
server/web2py/applications/vidjil/controllers/clonedb.py
View file @
3041dbc5
...
...
@@ -36,37 +36,49 @@ def search_clonedb(sequences, sample_set_id):
results
=
[]
parent_group
=
get_default_creation_group
(
auth
)[
1
]
for
sequence
in
sequences
:
options
=
clonedb
.
build_grep_clones_options
({
'sequence'
:
sequence
+
' -sample_set:%d'
%
sample_set_id
,
auth
.
load_permissions
(
PermissionEnum
.
read
.
value
,
'sample_set'
)
auth
.
load_permissions
(
PermissionEnum
.
anon
.
value
,
'sample_set'
)
options
=
clonedb
.
build_grep_clones_options
({
'sequence'
:
sequences
[
0
]
+
' -sample_set:%d'
%
sample_set_id
,
'index'
:
'clonedb_{}'
.
format
(
parent_group
)})
args
=
grep_clones
.
parser
.
parse_args
(
options
)
try
:
occurrences
=
grep_clones
.
launch_search
(
args
)
except
ValueError
:
return
error_message
(
'Are you sure your account has an enabled CloneDB?'
)
except
Exception
as
e
:
return
error_message
(
e
.
message
)
for
occ
in
occurrences
:
options
+=
sequences
[
1
:]
args
=
grep_clones
.
parser
.
parse_args
(
options
)
try
:
occurrences
=
grep_clones
.
launch_search
(
args
)
except
ValueError
:
return
error_message
(
'Are you sure your account has an enabled CloneDB?'
)
except
Exception
as
e
:
return
error_message
(
e
.
message
)
sample_set_ids
=
[
sid
for
occurrences_one_seq
in
occurrences
for
occ
in
occurrences_one_seq
if
'tags'
in
occ
and
'sample_set'
in
occ
[
'tags'
]
for
sid
in
occ
[
'tags'
][
'sample_set'
]
]
sample_sets
=
SampleSets
(
sample_set_ids
)
sample_names
=
sample_sets
.
get_names
()
sample_tags
=
sample_sets
.
get_tag_names
()
for
occurrences_one_seq
in
occurrences
:
for
occ
in
occurrences_one_seq
:
if
'tags'
in
occ
and
'sample_set'
in
occ
[
'tags'
]:
info
=
get_info_of_viewable_sample_set
([
int
(
sample_id
)
for
sample_id
in
occ
[
'tags'
][
'sample_set'
]],
int
(
occ
[
'tags'
][
'config_id'
][
0
]))
info
=
get_info_of_viewable_sample_set
([
int
(
sample_id
)
for
sample_id
in
occ
[
'tags'
][
'sample_set'
]],
int
(
occ
[
'tags'
][
'config_id'
][
0
]),
sample_names
,
sample_tags
)
occ
[
'tags'
][
'sample_set_viewable'
]
=
info
[
'viewable'
]
occ
[
'tags'
][
'sample_set_name'
]
=
info
[
'name'
]
occ
[
'tags'
][
'sample_tags'
]
=
info
[
'sample_tags'
]
config_db
=
db
.
config
[
occ
[
'tags'
][
'config_id'
][
0
]]
occ
[
'tags'
][
'config_name'
]
=
[
config_db
.
name
if
config_db
else
None
]
results
.
append
(
occurrences
)
results
.
append
(
occurrences
_one_seq
)
return
response
.
json
(
results
)
def
get_info_of_viewable_sample_set
(
sample_sets
,
config
):
def
get_info_of_viewable_sample_set
(
sample_sets
,
config
,
sample_names
,
sample_tags
):
info
=
{
'viewable'
:
[],
'name'
:
[],
'sample_tags'
:
[]}
for
sample_id
in
sample_sets
:
viewable
=
auth
.
can_view_sample_set
(
sample_id
,
auth
.
user
)
viewable
=
auth
.
can_view_sample_set
(
sample_id
,
auth
.
user
.
id
)
info
[
'viewable'
].
append
(
viewable
)
if
viewable
:
info
[
'name'
].
append
(
get_sample_name
(
sample_id
))
tags
=
get_sample_set_tags
(
sample_id
)
for
row
in
tags
:
info
[
'sample_tags'
].
append
(
"#"
+
row
.
name
)
info
[
'name'
].
append
(
sample_names
.
get
(
sample_id
))
tags
=
sample_tags
.
get
(
sample_id
)
if
tags
:
for
row
in
tags
:
info
[
'sample_tags'
].
append
(
"#"
+
row
.
name
)
else
:
info
[
'name'
].
append
(
None
)
return
info
server/web2py/applications/vidjil/controllers/default.py
View file @
3041dbc5
...
...
@@ -260,7 +260,7 @@ def get_data():
if
(
sample_set
.
sample_type
==
defs
.
SET_TYPE_PATIENT
):
for
row
in
db
(
db
.
patient
.
sample_set_id
==
request
.
vars
[
"sample_set_id"
]
).
select
()
:
log_reference_id
=
row
.
id
patient_name
=
vidjil_utils
.
anon_ids
(
row
.
id
)
patient_name
=
vidjil_utils
.
anon_ids
(
[
row
.
id
])[
0
]
data
[
"dataFileName"
]
=
patient_name
+
" ("
+
config_name
+
")"
data
[
"info"
]
=
db
.
patient
[
row
.
id
].
info
data
[
"patient_id"
]
=
row
.
id
...
...
@@ -401,7 +401,7 @@ def get_custom_data():
patient_run
=
db
(
db
[
sample_set
.
sample_type
].
sample_set_id
==
sample_set
.
id
).
select
().
first
()
config_id
=
db
.
results_file
[
id
].
config_id
name
=
vidjil_utils
.
anon_ids
(
patient_run
.
id
)
if
sample_set
.
sample_type
==
defs
.
SET_TYPE_PATIENT
else
patient_run
.
name
name
=
vidjil_utils
.
anon_ids
(
[
patient_run
.
id
])[
0
]
if
sample_set
.
sample_type
==
defs
.
SET_TYPE_PATIENT
else
patient_run
.
name
filename
=
db
.
sequence_file
[
sequence_file_id
].
filename
data
[
"samples"
][
"original_names"
].
append
(
name
+
"_"
+
filename
)
data
[
"samples"
][
"timestamp"
].
append
(
str
(
db
.
sequence_file
[
sequence_file_id
].
sampling_date
))
...
...
server/web2py/applications/vidjil/controllers/log.py
View file @
3041dbc5
...
...
@@ -13,7 +13,7 @@ if request.env.http_origin:
def
anon_names
(
data
):
for
row
in
data
:
# TODO use helper ?
row
.
name
=
vidjil_utils
.
anon_ids
(
row
.
id
)
row
.
name
=
vidjil_utils
.
anon_ids
(
[
row
.
id
])[
0
]
return
data
def
get_data_list
(
table
):
...
...
@@ -72,7 +72,7 @@ def index():
orderby
=~
db
.
user_log
.
created
)
for
row
in
query
:
if
row
.
patient
.
first_name
is
not
None
:
row
.
names
=
vidjil_utils
.
anon_ids
(
row
.
user_log
.
record_id
)
row
.
names
=
vidjil_utils
.
anon_ids
(
[
row
.
user_log
.
record_id
])[
0
]
else
:
row
.
names
=
row
.
run
.
name
return
dict
(
query
=
query
,
...
...
server/web2py/applications/vidjil/models/SampleSet.py
View file @
3041dbc5
...
...
@@ -162,7 +162,7 @@ def get_sample_name(sample_set_id):
if
patient_or_run
is
None
:
return
None
if
sample
.
sample_type
==
defs
.
SET_TYPE_PATIENT
:
return
vidjil_utils
.
anon_ids
(
patient_or_run
.
id
)
return
vidjil_utils
.
anon_ids
(
[
patient_or_run
.
id
])[
0
]
return
patient_or_run
.
name
def
get_set_group
(
sid
):
...
...
server/web2py/applications/vidjil/models/SampleSets.py
0 → 100644
View file @
3041dbc5
from
collections
import
defaultdict
import
vidjil_utils
class
SampleSets
:
'''
The SampleSets class represent many sample sets, possibly with different types.
This class allows to recover information on sample sets more efficiently than
by retrieving information for each sample set separately.
'''
ids
=
[]
sample_types
=
set
()
# All the sample typs we have in our IDs
sample_sets
=
[]
def
__init__
(
self
,
ids
):
'''
Build a class for the given sample set IDs
'''
self
.
ids
=
ids
self
.
sample_sets
=
db
(
db
.
sample_set
.
id
.
belongs
(
ids
)).
select
(
db
.
sample_set
.
sample_type
,
db
.
sample_set
.
id
)
self
.
sample_types
=
set
([
s
.
sample_type
for
s
in
self
.
sample_sets
])
def
get_names
(
self
):
'''Returns the names of all the sample sets.
The function returns a dictionary whose keys are sample set
IDs and value is the name.
The number of queries is constant (which is true only if
permissions have been cached: anon_ids checks for permission)
'''
results
=
{}
for
stype
in
self
.
sample_types
:
stype_ids
=
self
.
get_sample_type_ids
(
stype
)
if
stype
==
defs
.
SET_TYPE_PATIENT
:
patient_ids
=
db
(
db
[
stype
].
sample_set_id
.
belongs
(
stype_ids
)).
select
(
db
[
stype
].
id
,
db
[
stype
].
sample_set_id
)
names
=
vidjil_utils
.
anon_ids
([
p
.
id
for
p
in
patient_ids
])
for
i
,
p_id
in
enumerate
(
patient_ids
):
results
[
p_id
.
sample_set_id
]
=
names
[
i
]
else
:
set_names
=
db
(
db
[
stype
].
sample_set_id
.
belongs
(
stype_ids
)).
select
(
db
[
stype
].
sample_set_id
,
db
[
stype
].
name
)
for
s
in
set_names
:
results
[
s
.
sample_set_id
]
=
s
.
name
return
results
def
get_sample_type_ids
(
self
,
sample_type
):
'''
Return a list of all the sample set IDs corresponding to this sample type.
Performs no query on the database
'''
return
[
s
.
id
for
s
in
self
.
sample_sets
if
s
.
sample_type
==
sample_type
]
def
get_tag_names
(
self
):
'''Returns the name of the tags associated with all the sample sets.
The function returns a dictionary whose keys are sample set
IDs and values are a list of tag names.
The number of queries is constant.
'''
# Getting tags
results
=
defaultdict
(
list
)
for
stype
in
self
.
sample_types
:
stype_ids
=
self
.
get_sample_type_ids
(
stype
)
tags
=
db
((
db
[
stype
].
sample_set_id
.
belongs
(
stype_ids
))
&
(
db
.
tag_ref
.
record_id
==
db
[
stype
].
id
)
&
(
db
.
tag_ref
.
table_name
==
stype
)
&
(
db
.
tag_ref
.
tag_id
==
db
.
tag
.
id
))
\
.
select
(
db
.
tag
.
name
,
db
[
stype
].
sample_set_id
)
for
t
in
tags
:
results
[
t
[
stype
].
sample_set_id
].
append
(
t
.
tag
.
name
)
return
results
server/web2py/applications/vidjil/models/VidjilAuth.py
View file @
3041dbc5
...
...
@@ -446,14 +446,17 @@ class VidjilAuth(Auth):
or
self
.
is_admin
(
user
))
def
can_view_sample_set
(
self
,
sample_set_id
,
user
=
None
)
:
perm
=
self
.
get_permission
(
PermissionEnum
.
read
.
value
,
'sample_set'
,
sample_set_id
,
user
)
\
or
self
.
is_admin
(
user
)
if
perm
:
return
perm
sample_set
=
db
.
sample_set
[
sample_set_id
]
sample_type
=
sample_set
.
sample_type
if
sample_set
is
None
:
return
False
perm
=
self
.
get_permission
(
PermissionEnum
.
read
.
value
,
'sample_set'
,
sample_set_id
,
user
)
\
or
self
.
is_admin
(
user
)
for
row
in
db
(
db
[
sample_type
].
sample_set_id
==
sample_set_id
).
select
()
:
if
self
.
can_view
(
sample_type
,
row
.
id
,
user
):
perm
=
True
;
...
...
server/web2py/applications/vidjil/modules/vidjil_utils.py
View file @
3041dbc5
...
...
@@ -87,14 +87,18 @@ def anon_birth(patient_id, user_id):
else
:
return
age
def
anon_ids
(
patient_id
):
'''Anonymize patient name. Only the 'anon' access see the full patient name.'''
def
anon_ids
(
patient_ids
,
can_view
=
None
):
'''Anonymize patient name. Only the 'anon' access see the full patient name.
patient_ids is a list of patient IDs
'''
db
=
current
.
db
auth
=
current
.
auth
patient
=
db
.
patient
[
patient_id
]
patients
=
db
(
db
.
patient
.
id
.
belongs
(
patient_ids
)).
select
(
db
.
patient
.
sample_set_id
,
db
.
patient
.
first_name
,
db
.
patient
.
last_name
)
return
display_names
(
p
atient
.
sample_set_id
,
p
atient
.
first_name
,
p
atient
.
last_name
)
return
[
display_names
(
p
.
sample_set_id
,
p
.
first_name
,
p
.
last_name
,
can_view
)
for
p
in
patients
]
def
anon_names
(
sample_set_id
,
first_name
,
last_name
,
can_view
=
None
):
'''
...
...
server/web2py/applications/vidjil/tests/unit/models/SampleSets.py
0 → 100644
View file @
3041dbc5
#!/usr/bin/python
import
unittest
class
SamplesetsModel
(
unittest
.
TestCase
):
def
__init__
(
self
,
p
):
global
auth
unittest
.
TestCase
.
__init__
(
self
,
p
)
def
setUp
(
self
):
# Load the to-be-tested file
execfile
(
"applications/vidjil/models/SampleSets.py"
,
globals
())
global
auth
auth
=
VidjilAuth
(
globals
(),
db
)
auth
.
login_bare
(
"test@vidjil.org"
,
"1234"
)
# We have the following sample sets
# fake_sample_set_id linked to fake_patient_id
# whose first and last name are plop
# permission_sample_set linked to permission_patient
# whose name is foo bar
def
testGetNames
(
self
):
samples
=
SampleSets
([
fake_sample_set_id
,
permission_sample_set
])
names
=
samples
.
get_names
()
self
.
assertEquals
(
names
[
fake_sample_set_id
],
"plo"
)
self
.
assertEquals
(
names
[
permission_sample_set
],
"bar"
)
def
testGetTagNames
(
self
):
samples
=
SampleSets
([
fake_sample_set_id
,
permission_sample_set
])
names
=
samples
.
get_tag_names
()
self
.
assertEquals
(
'first_fake_tag'
in
names
[
fake_sample_set_id
],
True
)
self
.
assertEquals
(
'sec_fake_tag'
in
names
[
fake_sample_set_id
],
True
)
self
.
assertEquals
(
permission_sample_set
in
names
,
False
)
server/web2py/testRunner.py
View file @
3041dbc5
...
...
@@ -63,6 +63,8 @@ for tablename in db.tables: # Copy tables!
db
=
test_db
auth
.
db
=
test_db
current
.
db
=
db
current
.
auth
=
auth
# build default database if doesn't exist
vidjil_utils
.
init_db_helper
(
db
,
auth
,
True
)
...
...
@@ -90,6 +92,8 @@ user_id = db.auth_user.insert(
unique_group
=
db
.
auth_group
.
insert
(
role
=
"user_"
+
str
(
user_id
),
description
=
" "
)
db
.
auth_membership
.
insert
(
user_id
=
user_id
,
group_id
=
unique_group
)
auth
.
user
=
db
.
auth_user
[
user_id
]
# with admin privilege
group_id
=
1
#admin group
db
.
auth_membership
.
insert
(
user_id
=
user_id
,
group_id
=
group_id
)
...
...
@@ -237,6 +241,13 @@ sec_fake_tag_id = db.tag.insert(name="sec_fake_tag")
db
.
group_tag
.
insert
(
group_id
=
unique_group
,
tag_id
=
first_fake_tag_id
)
db
.
group_tag
.
insert
(
group_id
=
fake_group_id
,
tag_id
=
sec_fake_tag_id
)
first_fake_tag_ref
=
db
.
tag_ref
.
insert
(
tag_id
=
first_fake_tag_id
,
table_name
=
'patient'
,
record_id
=
fake_sample_set_id
)
sec_fake_tag_ref
=
db
.
tag_ref
.
insert
(
tag_id
=
sec_fake_tag_id
,
table_name
=
'patient'
,
record_id
=
fake_sample_set_id
)
db
.
commit
()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment