Update of the migration process

We decided to have all the tables in a unique database. In order to
perform this requirement I rewrote the migration files to duplicate all
the tables under a new name prefixed by `dj_` for the allgo tables. The
Django mandatory tables haven't been changed. The current model has been
adapted to respect this new mapping.

Signed-off-by: Matthieu Berjon <matthieu.berjon@inria.fr>

django/allgo/settings.py

@@ -80,7 +80,7 @@ WSGI_APPLICATION = 'allgo.wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'allgopy',
+        'NAME': 'allgo',
         'USER': 'allgo',
         'HOST': 'localhost',
     }

django/poc/models.py

@@ -31,7 +31,7 @@ class AllgoUser(models.Model):
             User, on_delete=models.CASCADE, related_name="allgouser")
 
     class Meta:
-        db_table = 'allgo_user'
+        db_table = 'dj_users'
 
 
 class DockerOs(TimeStampModel):
@@ -46,7 +46,7 @@ class DockerOs(TimeStampModel):
     docker_name = models.CharField(max_length=255, blank=True)
 
     class Meta:
-        db_table = 'docker_os'
+        db_table = 'dj_docker_os'
 
 
 class JobQueue(TimeStampModel):
@@ -60,7 +60,7 @@ class JobQueue(TimeStampModel):
     is_default = models.IntegerField()
 
     class Meta:
-        db_table = 'job_queues'
+        db_table = 'dj_job_queues'
 
 
 class Webapp(TimeStampModel):
@@ -106,11 +106,11 @@ class Webapp(TimeStampModel):
     docker_os = models.OneToOneField(DockerOs, related_name="webappdockeros")
     user = models.ForeignKey(
             User, on_delete=models.CASCADE, related_name="webappuser")
-    job_queue = models.OneToOneField(
+    dj_job_queue = models.OneToOneField(
             JobQueue, on_delete=models.CASCADE, related_name="webappjobqueue")
 
     class Meta:
-        db_table = 'webapps'
+        db_table = 'dj_webapps'
 
 
 class WebappParameter(TimeStampModel):
@@ -128,7 +128,7 @@ class WebappParameter(TimeStampModel):
     webapp = models.OneToOneField(Webapp, related_name="webappparameters")
 
     class Meta:
-        db_table = 'webapp_parameters'
+        db_table = 'dj_webapp_parameters'
 
 
 class WebappVersion(TimeStampModel):
@@ -144,10 +144,10 @@ class WebappVersion(TimeStampModel):
     published = models.IntegerField()
 
     # Relationships
-    webapp = models.ForeignKey(Webapp, related_name="webappversion")
+    dj_webapp = models.ForeignKey(Webapp, related_name="webappversion")
 
     class Meta:
-        db_table = 'webapp_versions'
+        db_table = 'dj_webapp_versions'
 
 
 class Quota(TimeStampModel):
@@ -162,13 +162,13 @@ class Quota(TimeStampModel):
     quantity = models.BigIntegerField(blank=True, null=True)
 
     # Relationships
-    user = models.ForeignKey(
+    dj_user = models.ForeignKey(
             User, on_delete=models.CASCADE, related_name="quotauser")
-    webapp = models.ForeignKey(
+    dj_webapp = models.ForeignKey(
             Webapp, on_delete=models.CASCADE, related_name="quotawebapp")
 
     class Meta:
-        db_table = 'quotas'
+        db_table = 'dj_quotas'
 
 
 class Job(TimeStampModel):
@@ -186,15 +186,15 @@ class Job(TimeStampModel):
     result = models.IntegerField()
 
     # Relationships
-    queue = models.ForeignKey(JobQueue, related_name="jobqueue")
-    webapp = models.ForeignKey(Webapp, related_name="jobwebapp")
-    user = models.ForeignKey(AllgoUser, related_name="jobuser")
+    dj_queue = models.ForeignKey(JobQueue, related_name="jobqueue")
+    dj_webapp = models.ForeignKey(Webapp, related_name="jobwebapp")
+    dj_user = models.ForeignKey(AllgoUser, related_name="jobuser")
 
     # Not in use because I don't have any related table yet
-    container_id = models.IntegerField(blank=True, null=True)
+    dj_container_id = models.IntegerField(blank=True, null=True)
 
     class Meta:
-        db_table = 'jobs'
+        db_table = 'dj_jobs'
 
 
 class JobUploads(TimeStampModel):
@@ -210,7 +210,7 @@ class JobUploads(TimeStampModel):
             blank=True, auto_now_add=True, null=True)
 
     # Relationships
-    job = models.ForeignKey(Job, related_name="jobuploadjob")
+    dj_job = models.ForeignKey(Job, related_name="jobuploadjob")
 
     class Meta:
-        db_table = 'job_uploads'
+        db_table = 'dj_job_uploads'

django/tools/migration.sh

 #!/bin/sh
 #
 
-USER=root
-DB_SRC=allgo
-DB_TGT=allgopy
 TEMP_DIR=/tmp/sql
-TABLES="docker_os webapps webapp_parameters webapp_versions users quotas \
+RAILS_TABLES="docker_os webapps webapp_parameters webapp_versions users quotas \
   job_queues jobs job_uploads"
+DJANGO_TABLES="dj_docker_os dj_webapps dj_webapp_parameters dj_webapp_versions \
+  dj_users dj_quotas dj_job_queues dj_jobs dj_job_uploads auth_group \
+  auth_group_permissions auth_permission auth_user auth_user_groups \
+  auth_user_user_permissions django_admin_log django_content_type \
+  django_migrations django_session"
+
+
+ParseArgs () {
+  if [ $# -eq 3 ]; then
+    PASS=
+    USER=$1
+    HOST=$2
+    DB=$3
+  else
+    PASS="--password=$2"
+    USER=$1
+    HOST=$3
+    DB=$4
+  fi
+}
 
 
 DumpDb () {
   # Dump the table without the creation schema
   # using the argument -n and -t
   mkdir -p $TEMP_DIR
-  for t in $TABLES
+  for TABLE in $RAILS_TABLES
   do
-    mysqldump -u $USER -h $1 $DB_SRC $t > $TEMP_DIR/db_$t.sql
+    mysqldump -u $USER $PASS -h $HOST $DB $TABLE > $TEMP_DIR/db_$TABLE.sql
   done
 }
 
 
-LoadDb () {
-  # We have to first get all the tables (including the ones created by Django)
-  TABLES_TO_DROP=$(mysql -u $USER -h $1 $DB_TGT -e 'SHOW TABLES;')
-  TO_DROP=$(echo $TABLES_TO_DROP | awk '{for(i=2;i<=NF;i++)print $i}')
-
-  if [ ${#TO_DROP[@]} -gt 0 ]; then
-    for TABLE in $TO_DROP
-    do
-      # We disable the foreign key constraint check in order to avoid dropping
-      # issues. We reactivate it just afterwards
-      mysql -u $USER -h $1 $DB_TGT -e \
-        "SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $TABLE; SET FOREIGN_KEY_CHECKS = 1;"
-    done
-  fi
+DeleteDjangoTables () {
+  for TABLE in $DJANGO_TABLES
+  do
+    mysql -u $USER $PASS -h $HOST $DB -e \
+      "SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $TABLE; SET FOREIGN_KEY_CHECKS = 1;"
+  done
+}
+
 
+LoadDb () {
   # Use Django to recreate the tables
   # Ensure that the virtual environment is activated
   # 
   # In production, there is no need of virtual environment
   rm -rf poc/migrations
 
-  for TABLE in $TABLES
-  do
-    mysql -u $USER -h $1 $DB_TGT < $TEMP_DIR/db_$TABLE.sql
-  done
-
   # Apply the migration
-  mysql -u $USER -h $1 $DB_TGT < tools/migration2django.sql
+  mysql -u $USER $PASS -h $HOST $DB< tools/migration2django.sql
   
   # Tell Django to use the current status of the database and start the migrations
   # from that point
@@ -60,31 +67,29 @@ Help() {
   cat << EOF
 Use: $0 [options]
 Options:
-  -d, --dump HOST       Dump the tables of the allgo database into the sql
-                        folder
-  -i, --init HOST       Dump the tables of the allgo database, make the
-                        migration and delete the dump (-d and -l at the same
-                        time)
-  -l, --load HOST       Load the tables into the allgopy database
-  -h, --help            Show this help
+  -d, --delete-django-tables USER [PASSWORD] HOST DB    Delete the Django tables
+  -e, --extract-rails-db USER [PASSWORD] HOST DB      Dump the tables of the allgo database into the sql folder
+  -m, --migrate-to-django USER [PASSWORD] HOST DB      Load the tables into the allgopy database
+  -h, --help                                    Show this help
 EOF
 }
 
-while getopts d:i:l:h OPTIONS
+while getopts d:e:m:h OPTIONS
 do
   case $OPTIONS in
 
-    d | --dump)
-      DumpDb $OPTARG
-      ;;
-
-    i | --init)
-      DumpDb $OPTARG
-      LoadDb $OPTARG
-      rm -rf $TEMP_DIR
+    d | --delete-django-tables)
+      ParseArgs $2 $3 $4 $5
+      DeleteDjangoTables 
     ;;
 
-    l | --load)
+    e | --extract-rails-db)
+      ParseArgs $2 $3 $4 $5
+      DumpDb
+      ;;
+
+    m | --migrate-to-django)
+      ParseArgs $2 $3 $4 $5
       LoadDb $OPTARG
       ;;
 
@@ -94,5 +99,3 @@ do
 
   esac
 done
-
-

django/tools/migration2django.sql

--- Duplicate the 'users' table as 'auth_user' (django style)
+-- Duplicate all tables before applying any changes
 CREATE TABLE auth_user AS SELECT * FROM users;
+CREATE TABLE dj_users AS SELECT * FROM users;
+CREATE TABLE dj_quotas AS SELECT * FROM quotas;
+CREATE TABLE dj_docker_os AS SELECT * FROM docker_os;
+CREATE TABLE dj_webapps AS SELECT * FROM webapps;
+CREATE TABLE dj_webapp_versions AS SELECT * FROM webapp_versions;
+CREATE TABLE dj_webapp_parameters AS SELECT * FROM webapp_parameters;
+CREATE TABLE dj_jobs AS SELECT * FROM jobs;
+CREATE TABLE dj_job_queues AS SELECT * FROM job_queues;
+CREATE TABLE dj_job_uploads AS SELECT * FROM job_uploads;
 
 --
 -- Modify  the auth_user to fit Django's original table format
@@ -223,8 +232,7 @@ ALTER TABLE `auth_user_user_permissions`
 -- Update of users table
 --
 
-RENAME TABLE `users` TO `allgo_user`;
-ALTER TABLE `allgo_user`
+ALTER TABLE `dj_users`
   ADD `user_id` int(11) NOT NULL,
   DROP COLUMN `email`,
   DROP COLUMN `encrypted_password`,
@@ -243,23 +251,23 @@ ALTER TABLE `allgo_user`
   DROP COLUMN `confirmed_at`,
   DROP COLUMN `confirmation_sent_at`,
   DROP COLUMN `auth_token`;
-UPDATE `allgo_user` SET `user_id`=`id`;
-ALTER TABLE `allgo_user`
+UPDATE `dj_users` SET `user_id`=`id`;
+ALTER TABLE `dj_users`
   ADD UNIQUE KEY `user_id` (`user_id`);
-ALTER TABLE `allgo_user`
-  ADD CONSTRAINT `allgo_user_user_id_aa366e15_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
+ALTER TABLE `dj_users`
+  ADD CONSTRAINT `dj_users_users_id_aa366e15_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
 
 --
 -- Update of webapps table
 --
 
-ALTER TABLE `webapps` CHANGE COLUMN `default_job_queue_id` `job_queue_id` int(11) NOT NULL;
+ALTER TABLE `dj_webapps` CHANGE COLUMN `default_job_queue_id` `dj_job_queue_id` int(11) NOT NULL;
 
 --
 -- Update of jobs table
 --
 
-ALTER TABLE `jobs`
+ALTER TABLE `dj_jobs`
   DROP COLUMN `size_generated_data`,
   DROP COLUMN `ipaddress`;
 
@@ -267,7 +275,7 @@ ALTER TABLE `jobs`
 -- Update of jobs_uploads
 --
 
-Alter TABLE `job_uploads`
+Alter TABLE `dj_job_uploads`
   DROP COLUMN `type`;