Commit e9393969 authored by marc duez's avatar marc duez

Merge branch 'feature-s/fix_migrator_script' into 'dev'

Feature s/fix migrator script

See merge request !816
parents a5bf9153 04ac443c
Pipeline #175737 passed with stages
in 8 minutes and 49 seconds
......@@ -78,6 +78,8 @@ class IdMapper():
self.mapping = {}
def getMatchingId(self, oid):
if oid is None:
return oid
if oid not in self.mapping:
self.log.debug('id %d not in mapping, returning it' % oid)
self.log.debug("mapping: " + str(self.mapping.keys()))
......@@ -134,7 +136,8 @@ class Extractor():
memberships = {}
sequence_files = {}
for row in rows:
row.provider = 1
row.sequence_file.provider = 1
row.sequence_file.patient_id = None
ssm_id = row.sample_set_membership.id
sf_id = row.sequence_file.id
self.log.debug("populating sequence file: %d, membership: %d" % (sf_id, ssm_id))
......@@ -164,11 +167,9 @@ class GroupExtractor(Extractor):
def getAccessible(self, table, groupids):
db = self.db
rows = db((((db[table].id == db.auth_permission.record_id)
& (db.auth_permission.table_name == table))
| ((db.sample_set.id == db.auth_permission.record_id)
& (db.sample_set.id == db[table].sample_set_id)
& (db.auth_permission.table_name == "sample_set")))
rows = db((db.sample_set.id == db.auth_permission.record_id)
& (db.sample_set.id == db[table].sample_set_id)
& (db.auth_permission.table_name == "sample_set")
& (db.auth_permission.name == PermissionEnum.access.value)
& (db.auth_permission.group_id.belongs(groupids))
).select(db[table].ALL)
......@@ -186,12 +187,13 @@ class SampleSetExtractor(Extractor):
class Importer():
def __init__(self, groupid, db, log, config_mapper):
def __init__(self, groupid, db, log, config_mapper, pprocess_mapper):
self.log = log
self.log.info("initialising importer")
self.groupid = groupid
self.db = db
self.mappings = {'config': config_mapper}
self.mappings = {'config': config_mapper,
'pre_process': pprocess_mapper}
self.mappings['sample_set'] = IdMapper(self.log)
def importSampleSets(self, stype, sets):
......@@ -225,7 +227,7 @@ class Importer():
for key in ref_fields:
ref_key = ref_fields[key]
matching_id = self.mappings[key].getMatchingId(val[ref_key])
self.log.debug("%s replacing %s: %d with %d" % (table, ref_key, val[ref_key], matching_id))
#self.log.debug("%s replacing %s: %d with %d" % (table, ref_key, val[ref_key], matching_id))
val[ref_key] = matching_id
oid = db[table].insert(**val)
self.log.debug("new %s: %d" % (table, oid))
......@@ -325,7 +327,7 @@ def export_sample_set_data(filesrc, filepath, sample_type, sample_ids, log=Migra
log.info("done")
def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=MigrateLogger()):
def import_data(filesrc, filedest, groupid, config=None, pprocess=None, dry_run=False, log=MigrateLogger()):
log.info("importing data")
data = {}
with open(filesrc + '/export.json', 'r') as infile:
......@@ -335,7 +337,12 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
config_mapper = ConfigMapper(log)
if config:
config_mapper.load(config)
imp = Importer(groupid, db, log, config_mapper)
pprocess_mapper = ConfigMapper(log)
if pprocess_mapper:
pprocess_mapper.load(pprocess)
imp = Importer(groupid, db, log, config_mapper, pprocess_mapper)
try:
set_types = ['patient', 'run', 'generic']
......@@ -343,7 +350,14 @@ def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=Migr
if stype in data:
imp.importSampleSets(stype, data[stype])
imp.importTable('sequence_file', data['sequence_file'], map_val=True)
for row in data['analysis_file']:
data['analysis_file'][row]['patient_id'] = None
data['analysis_file'][row]['config_id'] = None
for row in data['fused_file']:
data['fused_file'][row]['patient_id'] = None
imp.importTable('sequence_file', data['sequence_file'], {'pre_process': 'pre_process_id'}, map_val=True)
imp.importTable('sample_set_membership', data['membership'], {'sample_set': 'sample_set_id', 'sequence_file': 'sequence_file_id'})
imp.importTable('scheduler_task', data['scheduler_task'], map_val=True)
imp.importTable('scheduler_run', data['scheduler_run'], {'scheduler_task': 'task_id'})
......@@ -382,6 +396,7 @@ def main():
import_parser = subparsers.add_parser('import', help='Import data from JSON into the DB')
import_parser.add_argument('--dry-run', dest='dry', action='store_true', help='With a dry run, the data will not be saved to the database')
import_parser.add_argument('--config', type=str, dest='config', help='Select the config mapping file')
import_parser.add_argument('--pre-process', type=str, dest='pprocess', help='Select the pre-process mapping file')
import_parser.add_argument('groupid', type=long, help='The long ID of the receiver group')
parser.add_argument('-p', type=str, dest='filepath', default='./', help='Select the file destination')
......@@ -402,7 +417,7 @@ def main():
elif args.command == 'import':
if args.dry:
log.log.setLevel(logging.DEBUG)
import_data(args.filesrc, args.filepath, args.groupid, args.config, args.dry, log)
import_data(args.filesrc, args.filepath, args.groupid, args.config, args.pprocess, args.dry, log)
if __name__ == '__main__':
main()
......@@ -17,7 +17,8 @@ class MigratorScript(unittest.TestCase):
test_group = db.auth_group.insert(role='testy')
exp = Extractor(db, log)
config_mapper = ConfigMapper(log)
imp = Importer(test_group, db, log, config_mapper)
pprocess_mapper = ConfigMapper(log)
imp = Importer(test_group, db, log, config_mapper, pprocess_mapper)
test_patient_ids = []
test_sample_set_ids = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment