Skip to content

GitLab

Commit a5d33e4a authored by BERJON Matthieu's avatar BERJON Matthieu
Browse files

Update of the migration process 



We decided to have all the tables in a unique database. In order to
perform this requirement I rewrote the migration files to duplicate all
the tables under a new name prefixed by `dj_` for the allgo tables. The
Django mandatory tables haven't been changed. The current model has been
adapted to respect this new mapping.
Signed-off-by: BERJON Matthieu's avatarMatthieu Berjon <matthieu.berjon@inria.fr>
parent 3878ee31
Hide whitespace changes
Inline Side-by-side
Showing with 85 additions and 74 deletions
django/allgo/settings.py
View file @ a5d33e4a
......@@ -80,7 +80,7 @@ WSGI_APPLICATION = 'allgo.wsgi.application'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'allgopy',
'NAME': 'allgo',
'USER': 'allgo',
'HOST': 'localhost',
}
......
django/poc/models.py
View file @ a5d33e4a
......@@ -31,7 +31,7 @@ class AllgoUser(models.Model):
User, on_delete=models.CASCADE, related_name="allgouser")
class Meta:
db_table = 'allgo_user'
db_table = 'dj_users'
class DockerOs(TimeStampModel):
......@@ -46,7 +46,7 @@ class DockerOs(TimeStampModel):
docker_name = models.CharField(max_length=255, blank=True)
class Meta:
db_table = 'docker_os'
db_table = 'dj_docker_os'
class JobQueue(TimeStampModel):
......@@ -60,7 +60,7 @@ class JobQueue(TimeStampModel):
is_default = models.IntegerField()
class Meta:
db_table = 'job_queues'
db_table = 'dj_job_queues'
class Webapp(TimeStampModel):
......@@ -106,11 +106,11 @@ class Webapp(TimeStampModel):
docker_os = models.OneToOneField(DockerOs, related_name="webappdockeros")
user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="webappuser")
job_queue = models.OneToOneField(
dj_job_queue = models.OneToOneField(
JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
class Meta:
db_table = 'webapps'
db_table = 'dj_webapps'
class WebappParameter(TimeStampModel):
......@@ -128,7 +128,7 @@ class WebappParameter(TimeStampModel):
webapp = models.OneToOneField(Webapp, related_name="webappparameters")
class Meta:
db_table = 'webapp_parameters'
db_table = 'dj_webapp_parameters'
class WebappVersion(TimeStampModel):
......@@ -144,10 +144,10 @@ class WebappVersion(TimeStampModel):
published = models.IntegerField()
# Relationships
webapp = models.ForeignKey(Webapp, related_name="webappversion")
dj_webapp = models.ForeignKey(Webapp, related_name="webappversion")
class Meta:
db_table = 'webapp_versions'
db_table = 'dj_webapp_versions'
class Quota(TimeStampModel):
......@@ -162,13 +162,13 @@ class Quota(TimeStampModel):
quantity = models.BigIntegerField(blank=True, null=True)
# Relationships
user = models.ForeignKey(
dj_user = models.ForeignKey(
User, on_delete=models.CASCADE, related_name="quotauser")
webapp = models.ForeignKey(
dj_webapp = models.ForeignKey(
Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
class Meta:
db_table = 'quotas'
db_table = 'dj_quotas'
class Job(TimeStampModel):
......@@ -186,15 +186,15 @@ class Job(TimeStampModel):
result = models.IntegerField()
# Relationships
queue = models.ForeignKey(JobQueue, related_name="jobqueue")
webapp = models.ForeignKey(Webapp, related_name="jobwebapp")
user = models.ForeignKey(AllgoUser, related_name="jobuser")
dj_queue = models.ForeignKey(JobQueue, related_name="jobqueue")
dj_webapp = models.ForeignKey(Webapp, related_name="jobwebapp")
dj_user = models.ForeignKey(AllgoUser, related_name="jobuser")
# Not in use because I don't have any related table yet
container_id = models.IntegerField(blank=True, null=True)
dj_container_id = models.IntegerField(blank=True, null=True)
class Meta:
db_table = 'jobs'
db_table = 'dj_jobs'
class JobUploads(TimeStampModel):
......@@ -210,7 +210,7 @@ class JobUploads(TimeStampModel):
blank=True, auto_now_add=True, null=True)
# Relationships
job = models.ForeignKey(Job, related_name="jobuploadjob")
dj_job = models.ForeignKey(Job, related_name="jobuploadjob")
class Meta:
db_table = 'job_uploads'
db_table = 'dj_job_uploads'
django/tools/migration.sh
View file @ a5d33e4a
#!/bin/sh
#
USER=root
DB_SRC=allgo
DB_TGT=allgopy
TEMP_DIR=/tmp/sql
TABLES="docker_os webapps webapp_parameters webapp_versions users quotas \
RAILS_TABLES="docker_os webapps webapp_parameters webapp_versions users quotas \
job_queues jobs job_uploads"
DJANGO_TABLES="dj_docker_os dj_webapps dj_webapp_parameters dj_webapp_versions \
dj_users dj_quotas dj_job_queues dj_jobs dj_job_uploads auth_group \
auth_group_permissions auth_permission auth_user auth_user_groups \
auth_user_user_permissions django_admin_log django_content_type \
django_migrations django_session"
ParseArgs () {
if [ $# -eq 3 ]; then
PASS=
USER=$1
HOST=$2
DB=$3
else
PASS="--password=$2"
USER=$1
HOST=$3
DB=$4
fi
}
DumpDb () {
# Dump the table without the creation schema
# using the argument -n and -t
mkdir -p $TEMP_DIR
for t in $TABLES
for TABLE in $RAILS_TABLES
do
mysqldump -u $USER -h $1 $DB_SRC $t > $TEMP_DIR/db_$t.sql
mysqldump -u $USER $PASS -h $HOST $DB $TABLE > $TEMP_DIR/db_$TABLE.sql
done
}
LoadDb () {
# We have to first get all the tables (including the ones created by Django)
TABLES_TO_DROP=$(mysql -u $USER -h $1 $DB_TGT -e 'SHOW TABLES;')
TO_DROP=$(echo $TABLES_TO_DROP | awk '{for(i=2;i<=NF;i++)print $i}')
if [ ${#TO_DROP[@]} -gt 0 ]; then
for TABLE in $TO_DROP
do
# We disable the foreign key constraint check in order to avoid dropping
# issues. We reactivate it just afterwards
mysql -u $USER -h $1 $DB_TGT -e \
"SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $TABLE; SET FOREIGN_KEY_CHECKS = 1;"
done
fi
DeleteDjangoTables () {
for TABLE in $DJANGO_TABLES
do
mysql -u $USER $PASS -h $HOST $DB -e \
"SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $TABLE; SET FOREIGN_KEY_CHECKS = 1;"
done
}
LoadDb () {
# Use Django to recreate the tables
# Ensure that the virtual environment is activated
#
# In production, there is no need of virtual environment
rm -rf poc/migrations
for TABLE in $TABLES
do
mysql -u $USER -h $1 $DB_TGT < $TEMP_DIR/db_$TABLE.sql
done
# Apply the migration
mysql -u $USER -h $1 $DB_TGT < tools/migration2django.sql
mysql -u $USER $PASS -h $HOST $DB< tools/migration2django.sql
# Tell Django to use the current status of the database and start the migrations
# from that point
......@@ -60,31 +67,29 @@ Help() {
cat << EOF
Use: $0 [options]
Options:
-d, --dump HOST Dump the tables of the allgo database into the sql
folder
-i, --init HOST Dump the tables of the allgo database, make the
migration and delete the dump (-d and -l at the same
time)
-l, --load HOST Load the tables into the allgopy database
-h, --help Show this help
-d, --delete-django-tables USER [PASSWORD] HOST DB Delete the Django tables
-e, --extract-rails-db USER [PASSWORD] HOST DB Dump the tables of the allgo database into the sql folder
-m, --migrate-to-django USER [PASSWORD] HOST DB Load the tables into the allgopy database
-h, --help Show this help
EOF
}
while getopts d:i:l:h OPTIONS
while getopts d:e:m:h OPTIONS
do
case $OPTIONS in
d | --dump)
DumpDb $OPTARG
;;
i | --init)
DumpDb $OPTARG
LoadDb $OPTARG
rm -rf $TEMP_DIR
d | --delete-django-tables)
ParseArgs $2 $3 $4 $5
DeleteDjangoTables
;;
l | --load)
e | --extract-rails-db)
ParseArgs $2 $3 $4 $5
DumpDb
;;
m | --migrate-to-django)
ParseArgs $2 $3 $4 $5
LoadDb $OPTARG
;;
......@@ -94,5 +99,3 @@ do
esac
done
django/tools/migration2django.sql
View file @ a5d33e4a
-- Duplicate the 'users' table as 'auth_user' (django style)
-- Duplicate all tables before applying any changes
CREATE TABLE auth_user AS SELECT * FROM users;
CREATE TABLE dj_users AS SELECT * FROM users;
CREATE TABLE dj_quotas AS SELECT * FROM quotas;
CREATE TABLE dj_docker_os AS SELECT * FROM docker_os;
CREATE TABLE dj_webapps AS SELECT * FROM webapps;
CREATE TABLE dj_webapp_versions AS SELECT * FROM webapp_versions;
CREATE TABLE dj_webapp_parameters AS SELECT * FROM webapp_parameters;
CREATE TABLE dj_jobs AS SELECT * FROM jobs;
CREATE TABLE dj_job_queues AS SELECT * FROM job_queues;
CREATE TABLE dj_job_uploads AS SELECT * FROM job_uploads;
--
-- Modify the auth_user to fit Django's original table format
......@@ -223,8 +232,7 @@ ALTER TABLE `auth_user_user_permissions`
-- Update of users table
--
RENAME TABLE `users` TO `allgo_user`;
ALTER TABLE `allgo_user`
ALTER TABLE `dj_users`
ADD `user_id` int(11) NOT NULL,
DROP COLUMN `email`,
DROP COLUMN `encrypted_password`,
......@@ -243,23 +251,23 @@ ALTER TABLE `allgo_user`
DROP COLUMN `confirmed_at`,
DROP COLUMN `confirmation_sent_at`,
DROP COLUMN `auth_token`;
UPDATE `allgo_user` SET `user_id`=`id`;
ALTER TABLE `allgo_user`
UPDATE `dj_users` SET `user_id`=`id`;
ALTER TABLE `dj_users`
ADD UNIQUE KEY `user_id` (`user_id`);
ALTER TABLE `allgo_user`
ADD CONSTRAINT `allgo_user_user_id_aa366e15_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
ALTER TABLE `dj_users`
ADD CONSTRAINT `dj_users_users_id_aa366e15_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
--
-- Update of webapps table
--
ALTER TABLE `webapps` CHANGE COLUMN `default_job_queue_id` `job_queue_id` int(11) NOT NULL;
ALTER TABLE `dj_webapps` CHANGE COLUMN `default_job_queue_id` `dj_job_queue_id` int(11) NOT NULL;
--
-- Update of jobs table
--
ALTER TABLE `jobs`
ALTER TABLE `dj_jobs`
DROP COLUMN `size_generated_data`,
DROP COLUMN `ipaddress`;
......@@ -267,7 +275,7 @@ ALTER TABLE `jobs`
-- Update of jobs_uploads
--
Alter TABLE `job_uploads`
Alter TABLE `dj_job_uploads`
DROP COLUMN `type`;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment