Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vidjil
vidjil
Commits
97075f09
Commit
97075f09
authored
Oct 08, 2020
by
marc duez
Browse files
migrator.py : add pre-process mapper
parent
4b7c3f73
Changes
1
Hide whitespace changes
Inline
Side-by-side
server/web2py/applications/vidjil/scripts/migrator.py
View file @
97075f09
...
...
@@ -187,12 +187,13 @@ class SampleSetExtractor(Extractor):
class
Importer
():
def
__init__
(
self
,
groupid
,
db
,
log
,
config_mapper
):
def
__init__
(
self
,
groupid
,
db
,
log
,
config_mapper
,
pprocess_mapper
):
self
.
log
=
log
self
.
log
.
info
(
"initialising importer"
)
self
.
groupid
=
groupid
self
.
db
=
db
self
.
mappings
=
{
'config'
:
config_mapper
}
self
.
mappings
=
{
'config'
:
config_mapper
,
'pre_process'
:
pprocess_mapper
}
self
.
mappings
[
'sample_set'
]
=
IdMapper
(
self
.
log
)
def
importSampleSets
(
self
,
stype
,
sets
):
...
...
@@ -326,7 +327,7 @@ def export_sample_set_data(filesrc, filepath, sample_type, sample_ids, log=Migra
log
.
info
(
"done"
)
def
import_data
(
filesrc
,
filedest
,
groupid
,
config
=
None
,
dry_run
=
False
,
log
=
MigrateLogger
()):
def
import_data
(
filesrc
,
filedest
,
groupid
,
config
=
None
,
pprocess
=
None
,
dry_run
=
False
,
log
=
MigrateLogger
()):
log
.
info
(
"importing data"
)
data
=
{}
with
open
(
filesrc
+
'/export.json'
,
'r'
)
as
infile
:
...
...
@@ -336,7 +337,12 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
config_mapper
=
ConfigMapper
(
log
)
if
config
:
config_mapper
.
load
(
config
)
imp
=
Importer
(
groupid
,
db
,
log
,
config_mapper
)
pprocess_mapper
=
ConfigMapper
(
log
)
if
pprocess_mapper
:
pprocess_mapper
.
load
(
pprocess
)
imp
=
Importer
(
groupid
,
db
,
log
,
config_mapper
,
pprocess_mapper
)
try
:
set_types
=
[
'patient'
,
'run'
,
'generic'
]
...
...
@@ -350,6 +356,8 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
for
row
in
data
[
'fused_file'
]:
data
[
'fused_file'
][
row
][
'patient_id'
]
=
None
imp
.
importTable
(
'sequence_file'
,
data
[
'sequence_file'
],
{
'pre_process'
:
'pre_process_id'
},
map_val
=
True
)
imp
.
importTable
(
'sample_set_membership'
,
data
[
'membership'
],
{
'sample_set'
:
'sample_set_id'
,
'sequence_file'
:
'sequence_file_id'
})
imp
.
importTable
(
'scheduler_task'
,
data
[
'scheduler_task'
],
map_val
=
True
)
imp
.
importTable
(
'scheduler_run'
,
data
[
'scheduler_run'
],
{
'scheduler_task'
:
'task_id'
})
...
...
@@ -388,6 +396,7 @@ def main():
import_parser
=
subparsers
.
add_parser
(
'import'
,
help
=
'Import data from JSON into the DB'
)
import_parser
.
add_argument
(
'--dry-run'
,
dest
=
'dry'
,
action
=
'store_true'
,
help
=
'With a dry run, the data will not be saved to the database'
)
import_parser
.
add_argument
(
'--config'
,
type
=
str
,
dest
=
'config'
,
help
=
'Select the config mapping file'
)
import_parser
.
add_argument
(
'--pre-process'
,
type
=
str
,
dest
=
'pprocess'
,
help
=
'Select the pre-process mapping file'
)
import_parser
.
add_argument
(
'groupid'
,
type
=
long
,
help
=
'The long ID of the receiver group'
)
parser
.
add_argument
(
'-p'
,
type
=
str
,
dest
=
'filepath'
,
default
=
'./'
,
help
=
'Select the file destination'
)
...
...
@@ -408,7 +417,7 @@ def main():
elif
args
.
command
==
'import'
:
if
args
.
dry
:
log
.
log
.
setLevel
(
logging
.
DEBUG
)
import_data
(
args
.
filesrc
,
args
.
filepath
,
args
.
groupid
,
args
.
config
,
args
.
dry
,
log
)
import_data
(
args
.
filesrc
,
args
.
filepath
,
args
.
groupid
,
args
.
config
,
args
.
pprocess
,
args
.
dry
,
log
)
if
__name__
==
'__main__'
:
main
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment