Update of the migration process

Thanks to Sébastien Campion I rewrote the migration update. Now a SQL
file takes care of migrating and transforming all the data in the new
database. The bash script takes care of:

- deleting any existing tables in the Django database
- dump the current rails database and import it in the Django one
- apply the migration using the SQL script
- run the fake migration to take into account the current status of the
Django database

Signed-off-by: Matthieu Berjon <matthieu.berjon@inria.fr>

django/tools/migrate-db.sh → django/tools/migration.sh

 #!/bin/sh
 #
-# Extract each table
 
 USER=root
 HOST=localhost
@@ -17,7 +16,7 @@ DumpDb () {
   mkdir -p $TEMP_DIR
   for t in $TABLES
   do
-    mysqldump -n -t --extended-insert=FALSE -u $USER -h $HOST $DB_SRC $t > $TEMP_DIR/db_$t.sql
+    mysqldump -u $USER -h $HOST $DB_SRC $t > $TEMP_DIR/db_$t.sql
   done
 }
 
@@ -27,30 +26,34 @@ LoadDb () {
   TABLES_TO_DROP=$(mysql -u $USER $DB_TGT -e 'SHOW TABLES;')
   TO_DROP=$(echo $TABLES_TO_DROP | awk '{for(i=2;i<=NF;i++)print $i}')
 
-  for t in $TO_DROP
-  do
-    # We disable the foreign key constraint check in order to avoid dropping
-    # issues. We reactivate it just afterwards
-    mysql -u $USER -h $HOST $DB_TGT -e \
-      "SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $t; SET FOREIGN_KEY_CHECKS = 1;"
-  done
+  if [ ${#TO_DROP[@]} -gt 0 ]; then
+    for TABLE in $TO_DROP
+    do
+      # We disable the foreign key constraint check in order to avoid dropping
+      # issues. We reactivate it just afterwards
+      mysql -u $USER -h $HOST $DB_TGT -e \
+        "SET FOREIGN_KEY_CHECKS = 0; DROP TABLE $TABLE; SET FOREIGN_KEY_CHECKS = 1;"
+    done
+  fi
 
   # Use Django to recreate the tables
   # Ensure that the virtual environment is activated
   # 
   # In production, there is no need of virtual environment
-  cd ..
-  python manage.py makemigrations auth
-  python manage.py migrate poc 0001_initial
-  python manage.py migrate auth
+  rm -rf poc/migrations
 
-  # Insert the data into the DB
-  for t in $TABLES
+  for TABLE in $TABLES
   do
-    mysql -u $USER -h $HOST $DB_TGT < $TEMP_DIR/db_$t.sql
+    mysql -u $USER -h $HOST $DB_TGT < $TEMP_DIR/db_$TABLE.sql
   done
 
-  # rm -rf $TEMP_DIR
+  # Apply the migration
+  mysql -u $USER -h $HOST $DB_TGT < tools/migration.sql
+  
+  # Tell Django to use the current status of the database and start the migrations
+  # from that point
+  python manage.py makemigrations auth poc
+  python manage.py migrate --fake
 }
 
 

django/tools/migration.sql

+-- Duplicate the 'users' table as 'auth_user' (django style)
+CREATE TABLE auth_user AS SELECT * FROM users;
+
+--
+-- Modify  the auth_user to fit Django's original table format
+--
+ALTER TABLE `auth_user` MODIFY COLUMN `email` VARCHAR(254) COLLATE utf8_bin NOT NULL;
+ALTER TABLE `auth_user` CHANGE COLUMN `encrypted_password` `password` VARCHAR(128) COLLATE utf8_bin NOT NULL;
+ALTER TABLE `auth_user` CHANGE COLUMN `created_at` `date_joined` datetime(6) NOT NULL;
+ALTER TABLE `auth_user`
+  DROP COLUMN `reset_password_token`,
+  DROP COLUMN `reset_password_sent_at`,
+  DROP COLUMN `remember_created_at`,
+  DROP COLUMN `sign_in_count`,
+  DROP COLUMN `current_sign_in_at`,
+  DROP COLUMN `last_sign_in_at`,
+  DROP COLUMN `current_sign_in_ip`,
+  DROP COLUMN `last_sign_in_ip`,
+  DROP COLUMN `updated_at`,
+  DROP COLUMN `admin`,
+  DROP COLUMN `confirmation_token`,
+  DROP COLUMN `confirmed_at`,
+  DROP COLUMN `confirmation_sent_at`,
+  DROP COLUMN `sshkey`,
+  DROP COLUMN `auth_token`;
+ALTER TABLE `auth_user`
+  ADD `last_login` datetime(6) DEFAULT NULL,
+  ADD `is_superuser` tinyint(1) NOT NULL,
+  ADD `username` varchar(150) NOT NULL,
+  ADD `first_name` varchar(30) COLLATE utf8_bin NOT NULL,
+  ADD `last_name` varchar(30) COLLATE utf8_bin NOT NULL,
+  ADD `is_staff` tinyint(1) NOT NULL,
+  ADD `is_active` tinyint(1) NOT NULL;
+UPDATE `auth_user` SET `username`=`email`;
+ALTER TABLE `auth_user`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `username` (`username`);
+
+--
+-- Create all other Django specific tables
+--
+
+--
+-- Table structure for table `django_content_type`
+--
+
+CREATE TABLE `django_content_type` (
+  `id` int(11) NOT NULL,
+  `app_label` varchar(100) NOT NULL,
+  `model` varchar(100) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `django_content_type`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `django_content_type_app_label_model_76bd3d3b_uniq` (`app_label`,`model`);
+
+ALTER TABLE `django_content_type`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+--
+-- Table structure for table `django_migrations`
+--
+
+CREATE TABLE `django_migrations` (
+  `id` int(11) NOT NULL,
+  `app` varchar(255) NOT NULL,
+  `name` varchar(255) NOT NULL,
+  `applied` datetime(6) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `django_migrations`
+  ADD PRIMARY KEY (`id`);
+
+ALTER TABLE `django_migrations`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+--
+-- Table structure for table `django_admin_log`
+--
+
+CREATE TABLE `django_admin_log` (
+  `id` int(11) NOT NULL,
+  `action_time` datetime(6) NOT NULL,
+  `object_id` longtext DEFAULT NULL,
+  `object_repr` varchar(200) NOT NULL,
+  `action_flag` smallint(5) UNSIGNED NOT NULL,
+  `change_message` longtext NOT NULL,
+  `content_type_id` int(11) DEFAULT NULL,
+  `user_id` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `django_admin_log`
+  ADD PRIMARY KEY (`id`),
+  ADD KEY `django_admin_log_content_type_id_c4bce8eb_fk_django_co` (`content_type_id`),
+  ADD KEY `django_admin_log_user_id_c564eba6_fk_auth_user_id` (`user_id`);
+
+ALTER TABLE `django_admin_log`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `django_admin_log`
+  ADD CONSTRAINT `django_admin_log_content_type_id_c4bce8eb_fk_django_co` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`),
+  ADD CONSTRAINT `django_admin_log_user_id_c564eba6_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
+
+--
+-- Table structure for table `django_session`
+--
+
+CREATE TABLE `django_session` (
+  `session_key` varchar(40) COLLATE utf8_bin NOT NULL,
+  `session_data` longtext COLLATE utf8_bin NOT NULL,
+  `expire_date` datetime(6) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+ALTER TABLE `django_session`
+  ADD PRIMARY KEY (`session_key`),
+  ADD KEY `django_session_expire_date_a5c62663` (`expire_date`);
+
+--
+-- Table structure for table `auth_group`
+--
+
+CREATE TABLE `auth_group` (
+  `id` int(11) NOT NULL,
+  `name` varchar(80) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `auth_group`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `name` (`name`);
+
+ALTER TABLE `auth_group`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+--
+-- Table structure for table `auth_permission`
+--
+
+CREATE TABLE `auth_permission` (
+  `id` int(11) NOT NULL,
+  `name` varchar(255) NOT NULL,
+  `content_type_id` int(11) NOT NULL,
+  `codename` varchar(100) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `auth_permission`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `auth_permission_content_type_id_codename_01ab375a_uniq` (`content_type_id`,`codename`);
+
+ALTER TABLE `auth_permission`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `auth_permission`
+  ADD CONSTRAINT `auth_permission_content_type_id_2f476e4b_fk_django_co` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`);
+
+--
+-- Table structure for table `auth_group_permissions`
+--
+
+CREATE TABLE `auth_group_permissions` (
+  `id` int(11) NOT NULL,
+  `group_id` int(11) NOT NULL,
+  `permission_id` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `auth_group_permissions`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `auth_group_permissions_group_id_permission_id_0cd325b0_uniq` (`group_id`,`permission_id`),
+  ADD KEY `auth_group_permissio_permission_id_84c5c92e_fk_auth_perm` (`permission_id`);
+
+ALTER TABLE `auth_group_permissions`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `auth_group_permissions`
+  ADD CONSTRAINT `auth_group_permissio_permission_id_84c5c92e_fk_auth_perm` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`),
+  ADD CONSTRAINT `auth_group_permissions_group_id_b120cbf9_fk_auth_group_id` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`);
+
+--
+-- Table structure for table `auth_user_groups`
+--
+
+CREATE TABLE `auth_user_groups` (
+  `id` int(11) NOT NULL,
+  `user_id` int(11) NOT NULL,
+  `group_id` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `auth_user_groups`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `auth_user_groups_user_id_group_id_94350c0c_uniq` (`user_id`,`group_id`),
+  ADD KEY `auth_user_groups_group_id_97559544_fk_auth_group_id` (`group_id`);
+
+ALTER TABLE `auth_user_groups`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `auth_user_groups`
+  ADD CONSTRAINT `auth_user_groups_group_id_97559544_fk_auth_group_id` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`),
+  ADD CONSTRAINT `auth_user_groups_user_id_6a12ed8b_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
+
+--
+-- Table structure for table `auth_user_user_permissions`
+--
+
+CREATE TABLE `auth_user_user_permissions` (
+  `id` int(11) NOT NULL,
+  `user_id` int(11) NOT NULL,
+  `permission_id` int(11) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+ALTER TABLE `auth_user_user_permissions`
+  ADD PRIMARY KEY (`id`),
+  ADD UNIQUE KEY `auth_user_user_permissions_user_id_permission_id_14a6b632_uniq` (`user_id`,`permission_id`),
+  ADD KEY `auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm` (`permission_id`);
+
+ALTER TABLE `auth_user_user_permissions`
+  MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+
+ALTER TABLE `auth_user_user_permissions`
+  ADD CONSTRAINT `auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`),
+  ADD CONSTRAINT `auth_user_user_permissions_user_id_a95ead1b_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
+
+
+--
+-- Update of users table
+--
+
+RENAME TABLE `users` TO `allgo_user`;
+ALTER TABLE `allgo_user`
+  ADD `user_id` int(11) NOT NULL,
+  DROP COLUMN `email`,
+  DROP COLUMN `encrypted_password`,
+  DROP COLUMN `reset_password_token`,
+  DROP COLUMN `reset_password_sent_at`,
+  DROP COLUMN `remember_created_at`,
+  DROP COLUMN `sign_in_count`,
+  DROP COLUMN `current_sign_in_at`,
+  DROP COLUMN `last_sign_in_at`,
+  DROP COLUMN `current_sign_in_ip`,
+  DROP COLUMN `last_sign_in_ip`,
+  DROP COLUMN `created_at`,
+  DROP COLUMN `updated_at`,
+  DROP COLUMN `admin`,
+  DROP COLUMN `confirmation_token`,
+  DROP COLUMN `confirmed_at`,
+  DROP COLUMN `confirmation_sent_at`,
+  DROP COLUMN `auth_token`;
+UPDATE `allgo_user` SET `user_id`=`id`;
+ALTER TABLE `allgo_user`
+  ADD UNIQUE KEY `user_id` (`user_id`);
+ALTER TABLE `allgo_user`
+  ADD CONSTRAINT `allgo_user_user_id_aa366e15_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`);
+
+--
+-- Update of webapps table
+--
+
+ALTER TABLE `webapps` CHANGE COLUMN `default_job_queue_id` `job_queue_id` int(11) NOT NULL;
+
+--
+-- Update of jobs table
+--
+
+ALTER TABLE `jobs`
+  DROP COLUMN `size_generated_data`,
+  DROP COLUMN `ipaddress`;
+
+--
+-- Update of jobs_uploads
+--
+
+Alter TABLE `job_uploads`
+  DROP COLUMN `type`;
+
+
+COMMIT;