Commit 97075f09 authored by marc duez's avatar marc duez
Browse files

migrator.py : add pre-process mapper

parent 4b7c3f73
...@@ -187,12 +187,13 @@ class SampleSetExtractor(Extractor): ...@@ -187,12 +187,13 @@ class SampleSetExtractor(Extractor):
class Importer(): class Importer():
def __init__(self, groupid, db, log, config_mapper): def __init__(self, groupid, db, log, config_mapper, pprocess_mapper):
self.log = log self.log = log
self.log.info("initialising importer") self.log.info("initialising importer")
self.groupid = groupid self.groupid = groupid
self.db = db self.db = db
self.mappings = {'config': config_mapper} self.mappings = {'config': config_mapper,
'pre_process': pprocess_mapper}
self.mappings['sample_set'] = IdMapper(self.log) self.mappings['sample_set'] = IdMapper(self.log)
def importSampleSets(self, stype, sets): def importSampleSets(self, stype, sets):
...@@ -326,7 +327,7 @@ def export_sample_set_data(filesrc, filepath, sample_type, sample_ids, log=Migra ...@@ -326,7 +327,7 @@ def export_sample_set_data(filesrc, filepath, sample_type, sample_ids, log=Migra
log.info("done") log.info("done")
def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=MigrateLogger()): def import_data(filesrc, filedest, groupid, config=None, pprocess=None, dry_run=False, log=MigrateLogger()):
log.info("importing data") log.info("importing data")
data = {} data = {}
with open(filesrc + '/export.json', 'r') as infile: with open(filesrc + '/export.json', 'r') as infile:
...@@ -336,7 +337,12 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr ...@@ -336,7 +337,12 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
config_mapper = ConfigMapper(log) config_mapper = ConfigMapper(log)
if config: if config:
config_mapper.load(config) config_mapper.load(config)
imp = Importer(groupid, db, log, config_mapper)
pprocess_mapper = ConfigMapper(log)
if pprocess_mapper:
pprocess_mapper.load(pprocess)
imp = Importer(groupid, db, log, config_mapper, pprocess_mapper)
try: try:
set_types = ['patient', 'run', 'generic'] set_types = ['patient', 'run', 'generic']
...@@ -350,6 +356,8 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr ...@@ -350,6 +356,8 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
for row in data['fused_file']: for row in data['fused_file']:
data['fused_file'][row]['patient_id'] = None data['fused_file'][row]['patient_id'] = None
imp.importTable('sequence_file', data['sequence_file'], {'pre_process': 'pre_process_id'}, map_val=True)
imp.importTable('sample_set_membership', data['membership'], {'sample_set': 'sample_set_id', 'sequence_file': 'sequence_file_id'}) imp.importTable('sample_set_membership', data['membership'], {'sample_set': 'sample_set_id', 'sequence_file': 'sequence_file_id'})
imp.importTable('scheduler_task', data['scheduler_task'], map_val=True) imp.importTable('scheduler_task', data['scheduler_task'], map_val=True)
imp.importTable('scheduler_run', data['scheduler_run'], {'scheduler_task': 'task_id'}) imp.importTable('scheduler_run', data['scheduler_run'], {'scheduler_task': 'task_id'})
...@@ -388,6 +396,7 @@ def main(): ...@@ -388,6 +396,7 @@ def main():
import_parser = subparsers.add_parser('import', help='Import data from JSON into the DB') import_parser = subparsers.add_parser('import', help='Import data from JSON into the DB')
import_parser.add_argument('--dry-run', dest='dry', action='store_true', help='With a dry run, the data will not be saved to the database') import_parser.add_argument('--dry-run', dest='dry', action='store_true', help='With a dry run, the data will not be saved to the database')
import_parser.add_argument('--config', type=str, dest='config', help='Select the config mapping file') import_parser.add_argument('--config', type=str, dest='config', help='Select the config mapping file')
import_parser.add_argument('--pre-process', type=str, dest='pprocess', help='Select the pre-process mapping file')
import_parser.add_argument('groupid', type=long, help='The long ID of the receiver group') import_parser.add_argument('groupid', type=long, help='The long ID of the receiver group')
parser.add_argument('-p', type=str, dest='filepath', default='./', help='Select the file destination') parser.add_argument('-p', type=str, dest='filepath', default='./', help='Select the file destination')
...@@ -408,7 +417,7 @@ def main(): ...@@ -408,7 +417,7 @@ def main():
elif args.command == 'import': elif args.command == 'import':
if args.dry: if args.dry:
log.log.setLevel(logging.DEBUG) log.log.setLevel(logging.DEBUG)
import_data(args.filesrc, args.filepath, args.groupid, args.config, args.dry, log) import_data(args.filesrc, args.filepath, args.groupid, args.config, args.pprocess, args.dry, log)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment