spatial transformer

.gitmodules

 [submodule "tensorflow"]
 	path = research/syntaxnet/tensorflow
 	url = https://github.com/tensorflow/tensorflow.git
+[submodule "research/spatial_transformer_networks"]
+	path = research/spatial_transformer_networks
+	url = https://github.com/oarriaga/spatial_transformer_networks.git

spatial_transformer_networks @ b80c50bb

+Subproject commit b80c50bb5f015fdf6eca28dab5ed9794ffa0a957

research/transformer/spatial_transformer.py

@@ -145,7 +145,7 @@ def transformer(U, theta, out_size, name='SpatialTransformer', **kwargs):
             num_batch = tf.shape(input_dim)[0]
             height = tf.shape(input_dim)[1]
             width = tf.shape(input_dim)[2]
-            num_channels = tf.shape(input_dim)[3]
+            num_channels = input_dim.shape[3]
             theta = tf.reshape(theta, (-1, 2, 3))
             theta = tf.cast(theta, 'float32')
 

research/transformer_detector/train.py

+import os
 import numpy as np
 import tensorflow as tf
+from sklearn.model_selection import StratifiedKFold, train_test_split
+from sklearn.preprocessing import LabelEncoder
+import keras
 from keras import backend as K
+from keras.optimizers import Adam
+from keras.callbacks import EarlyStopping, ModelCheckpoint
 
 from transformer_detector import TransformerDetector
 import logging
@@ -28,9 +34,11 @@ flags.DEFINE_string('dataset_path', '',
                     'Path to an npz file containing the dataset.')
 flags.DEFINE_boolean('loc_only', True,
                      'localization only')
+flags.DEFINE_boolean('nh', True, 'use note head centroid as input feature')
 flags.DEFINE_integer('num_classes', 4, 'number of classes')
 flags.DEFINE_string('log_file', '/tmp/test_log.log',
                     'log file path')
+flags.DEFINE_boolean('allow_growth', True, 'allow gpu memory growth')
 flags.DEFINE_string('gpu_device', '0',
                     'list of gpu to use')
 flags.DEFINE_float('memory_fraction', 1.0,
@@ -47,31 +55,103 @@ if K.backend() != 'tensorflow':
                        'are not supported on other platforms.')
 
 
+def filter_data(data):
+    return data
+
+
 def main():
     assert FLAGS.train_dir, '`train_dir` is missing.'
+    import autodebug
+    session_config = tf.ConfigProto(allow_soft_placement=True,
+                                    log_device_placement=False)
+    session_config.gpu_options.allow_growth = FLAGS.allow_growth
+    session_config.gpu_options.visible_device_list = FLAGS.gpu_device
+    session_config.gpu_options.per_process_gpu_memory_fraction = \
+        FLAGS.memory_fraction
+    K.set_session(tf.Session(config=session_config))
 
     detector = TransformerDetector(FLAGS.loc_only, FLAGS.num_classes)
 
     data = np.load(FLAGS.dataset_path)
+    labels = data['classnames']
+    le = LabelEncoder()
+    le.fit(list(set(labels)))
+    labels = le.transform(labels)
     images = data['images']
     images_shape = images.shape
     images_shape = [images_shape[0], images_shape[2], images_shape[3],
                     images_shape[1]]
     images = images.reshape(images_shape)
-    reltranss = data['reltranss']
-    reltranss = reltranss.reshape([reltranss.shape[0],
-                                   reltranss.shape[1] * reltranss.shape[2]])
-    tete_centroids = data['tete_centroids']
+    data_dict = {'image_input': images}
+    if FLAGS.nh:
+        tete_centroids = data['tete_centroids']
+        data_dict['anchor_point_input'] = tete_centroids
+    if FLAGS.loc_only:
+        output = data['reltranss']
+        output = output.reshape([output.shape[0],
+                                 output.shape[1] * output.shape[2]])
+    else:
+        output = keras.utils.to_categorical(labels)
     del data
-    data_dict = {'image_input': images,
-                 'anchor_point_input': tete_centroids}
-    model = detector.predict(data_dict)
 
     if FLAGS.loc_only:
-        model.compile(optimizer='adam', loss='mean_squared_error')
+        loss = 'mean_squared_error'
+        metrics = []
+        monitor = 'val_loss'
     else:
-        model.compile(optimizer='adam', loss='categorical_crossentropy')
-    model.fit(data_dict, reltranss, batch_size=50, epochs=2)
+        loss = 'categorical_crossentropy'
+        metrics = ['accuracy']
+        monitor = 'val_acc'
+    skf = StratifiedKFold(n_splits=5)
+    for i, (train_index, test_index) in enumerate(skf.split(
+            np.zeros(len(labels)), labels)):
+        model = detector.predict(data_dict)
+        model.compile(optimizer=Adam(lr=0.0001), loss=loss, metrics=metrics)
+        train_dir = os.path.join(FLAGS.train_dir, "fold_" + str(i))
+        callbacks = [
+            EarlyStopping(monitor=monitor, patience=10),
+            ModelCheckpoint(
+                os.path.join(train_dir,
+                             "weights.{epoch:02d}-{val_loss:.2f}.hdf5"),
+                monitor=monitor, save_best_only=True,
+                save_weights_only=True)]
+        if not os.path.exists(train_dir):
+            os.makedirs(train_dir)
+        train_image = data_dict['image_input'][train_index]
+        test_image = data_dict['image_input'][test_index]
+        if FLAGS.nh:
+            train_anchor_point = data_dict['anchor_point_input'][train_index]
+            test_anchor_point = data_dict['anchor_point_input'][test_index]
+        train_output = output[train_index]
+        test_output = output[test_index]
+        if FLAGS.nh:
+            train_image, val_image, train_anchor_point, val_anchor_point, \
+                train_output, val_output = train_test_split(
+                    train_image, train_anchor_point, train_output,
+                    test_size=0.3)
+            train_input_dict = {'image_input': train_image,
+                                'anchor_point_input': train_anchor_point}
+            val_input_dict = {'image_input': val_image,
+                              'anchor_point_input': val_anchor_point}
+        else:
+            train_image, val_image, train_output, val_output = \
+                train_test_split(train_image, train_output, test_size=0.3)
+            train_input_dict = {'image_input': train_image}
+            val_input_dict = {'image_input': val_image}
+        model.fit(train_input_dict, train_output, batch_size=50, epochs=500,
+                  callbacks=callbacks, validation_data=(val_input_dict,
+                                                        val_output))
+        if FLAGS.nh:
+            test_input_dict = {'image_input': test_image,
+                               'anchor_point_input': test_anchor_point}
+        else:
+            test_input_dict = {'image_input': test_image}
+        score = model.evaluate(test_input_dict, test_output)
+        if FLAGS.loc_only:
+            print('Test loss:', score)
+        else:
+            print('Test loss:', score[0])
+            print('Test accuracy:', score[1])
 
 
 if __name__ == "__main__":

research/transformer_detector/transformer_detector.py

@@ -67,8 +67,9 @@ class TransformerDetector(object):
         return model
 
     def _predict_class(self, preprocessed_inputs, image_input, loc_output):
-        x = transformer(image_input, loc_output,
-                        preprocessed_inputs['image_input'][1:].shape / 2)
+        new_shape = preprocessed_inputs['image_input'].shape[1:]
+        new_shape = (int(new_shape[0]/2), int(new_shape[1]/2))
+        x = transformer(image_input, loc_output, new_shape)
         x = Conv2D(32, (3, 3), kernel_initializer='he_uniform',
                    activation='relu', name='classif_feat_2')(x)
         x = MaxPooling2D((2, 2), name='classif_feat_3')(x)