{"cells":[{"metadata":{},"cell_type":"markdown","source":"ResNet50 train/validation accuracy is not converging\n\nMy code can be found here:\nhttps://github.com/evagian/Kaggle-Recursion-Cellular-Image-Classification\n\nYour help will be appreciated :)"},{"metadata":{"trusted":false},"cell_type":"code","source":"import os\n#os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\";\n#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\";\nfrom __future__ import print_function, division\nfrom builtins import range, input\n# Note: you may need to update your version of future\n# sudo pip install -U future\nimport theano\n\nfrom keras.layers import Input, Lambda, Dense, Flatten\nfrom keras import optimizers\nfrom keras.models import Model\nfrom keras.applications.resnet50 import ResNet50, preprocess_input\n# from keras.applications.inception_v3 import InceptionV3, preprocess_input\nfrom keras.preprocessing import image\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.metrics import confusion_matrix\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nprint(tf.__version__)\nfrom glob import glob\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Device mapping:\n#/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n#/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\nfrom tensorflow.python.client import device_lib\nprint(device_lib.list_local_devices())\n#sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# re-size all the images to this\nIMAGE_SIZE = [224, 224] # feel free to change depending on dataset\n\n# training config:\nepochs = 4\nbatch_size = 16\n\ntrain_path = '../input/train'\nvalid_path = '../input/test'\n\n# useful for getting number of files\nimage_files = glob(train_path + '/*/*.png')\nvalid_image_files = glob(valid_path + '/*/*.png')\n\n# useful for getting number of classes\nfolders = glob(train_path + '/*')\n\n# look at an image for fun\nplt.imshow(image.load_img(np.random.choice(image_files)))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# add preprocessing layer to the front of VGG\nres = ResNet50(input_shape=IMAGE_SIZE + [3], weights='imagenet', include_top=False)\n\n# don't train existing weights\nfor layer in res.layers:\n  layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# our layers - you can add more if you want\nx = Flatten()(res.output)\nx = Dense(1108, activation='relu')(x)\nx = Dense(1108, activation='relu')(x)\n\nprediction = Dense(len(folders), activation='softmax')(x)\n\n# create a model object\nmodel = Model(inputs=res.input, outputs=prediction)\n\n# view the structure of the model\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def random_crop(img, random_crop_size):\n    # Note: image_data_format is 'channel_last'\n    assert img.shape[2] == 3\n    height, width = img.shape[0], img.shape[1]\n    dy, dx = random_crop_size\n    x = np.random.randint(0, width - dx + 1)\n    y = np.random.randint(0, height - dy + 1)\n    return img[y:(y+dy), x:(x+dx), :]\n\n\ndef crop_generator(batches, crop_length):\n    \"\"\"Take as input a Keras ImageGen (Iterator) and generate random\n    crops from the image batches generated by the original iterator.\n    \"\"\"\n    while True:\n        batch_x, batch_y = next(batches)\n        batch_crops = np.zeros((batch_x.shape[0], crop_length, crop_length, 3))\n        for i in range(batch_x.shape[0]):\n            batch_crops[i] = random_crop(batch_x[i], (crop_length, crop_length))\n        yield (batch_crops, batch_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learning_rate = 0.05\ndecay_rate = learning_rate / epochs\nopt = optimizers.RMSprop(lr=learning_rate, decay=decay_rate)\n\n# tell the model what cost and optimization method to use\nmodel.compile(\n    optimizer=opt, \n    loss='categorical_crossentropy', \n    metrics=['accuracy']\n)\n\n# create an instance of ImageDataGenerator\ngen = ImageDataGenerator(\n  rotation_range=20,\n  width_shift_range=0.1,\n  height_shift_range=0.1,\n  shear_range=0.1,\n  zoom_range=0.2,\n  horizontal_flip=True,\n  vertical_flip=True,\n  preprocessing_function=preprocess_input\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# test generator to see how it works and some other useful things\n\n# get label mapping for confusion matrix plot later\ntest_gen = gen.flow_from_directory(valid_path, target_size=IMAGE_SIZE)\nprint(test_gen.class_indices)\nlabels = [None] * len(test_gen.class_indices)\nfor k, v in test_gen.class_indices.items():\n  labels[v] = k\n\n# should be a strangely colored image (due to VGG weights being BGR)\nfor x, y in test_gen:\n  print(\"min:\", x[0].min(), \"max:\", x[0].max())\n  plt.title(labels[np.argmax(y[0])])\n  plt.imshow(x[0])\n  plt.show()\n  break","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# create generators\ntrain_generator = gen.flow_from_directory(\n  train_path,\n  target_size=IMAGE_SIZE,\n  shuffle=True,\n  batch_size=batch_size,\n)\nvalid_generator = gen.flow_from_directory(\n  valid_path,\n  target_size=IMAGE_SIZE,\n  shuffle=True,\n  batch_size=batch_size,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"checkpoint_path = \"training1/cp.ckpt\"\ncheckpoint_dir = os.path.dirname(checkpoint_path)\n\n# Create checkpoint callback\ncp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n                                                 save_weights_only=True,\n                                                 verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class LearningRateScheduler(tf.keras.callbacks.Callback):\n  \"\"\"Learning rate scheduler which sets the learning rate according to schedule.\n\n  Arguments:\n      schedule: a function that takes an epoch index\n          (integer, indexed from 0) and current learning rate\n          as inputs and returns a new learning rate as output (float).\n  \"\"\"\n\n  def __init__(self, schedule):\n    super(LearningRateScheduler, self).__init__()\n    self.schedule = schedule\n\n  def on_epoch_begin(self, epoch, logs=None):\n    if not hasattr(self.model.optimizer, 'lr'):\n      raise ValueError('Optimizer must have a \"lr\" attribute.')\n    # Get the current learning rate from model's optimizer.\n    lr = float(tf.keras.backend.get_value(self.model.optimizer.lr))\n    # Call schedule function to get the scheduled learning rate.\n    scheduled_lr = self.schedule(epoch, lr)\n    # Set the value back to the optimizer before this epoch starts\n    tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)\n    print('\\nEpoch %05d: Learning rate is %6.4f.' % (epoch, scheduled_lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"LR_SCHEDULE = [\n    # (epoch to start, learning rate) tuples\n    (0, 0.05), (2, 0.01), (5, 0.005), (10, 0.001)\n]\n\ndef lr_schedule(epoch, lr):\n  \"\"\"Helper function to retrieve the scheduled learning rate based on epoch.\"\"\"\n  LR_SCHEDULE = [   (0, 0.05), (2, 0.01), (5, 0.005), (10, 0.001)]\n  if epoch < LR_SCHEDULE[0][0] or epoch > LR_SCHEDULE[-1][0]:\n    return lr\n  for i in range(len(LR_SCHEDULE)):\n    if epoch == LR_SCHEDULE[i][0]:\n      return LR_SCHEDULE[i][1]\n  return lr\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class LossAndErrorPrintingCallback(tf.keras.callbacks.Callback):\n\n  def on_train_batch_end(self, batch, logs=None):\n    print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n\n  def on_test_batch_end(self, batch, logs=None):\n    print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n\n  def on_epoch_end(self, epoch, logs=None):\n    print('The average loss for epoch {} is {:7.2f}.'.format(epoch, logs['loss']))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.load_weights(checkpoint_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_generator = crop_generator(train_generator, 224)\nvalid_generator = crop_generator(valid_generator, 224)\n\n# fit the model\nwith tf.device('/gpu:0'):\n\n    r = model.fit_generator(\n      train_generator,\n      validation_data=valid_generator,\n      initial_epoch=3,  \n      epochs=5,\n      steps_per_epoch=len(image_files) // batch_size,\n      validation_steps=len(valid_image_files) // batch_size,\n     callbacks = [cp_callback, LossAndErrorPrintingCallback()]\n    )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}