{"cells":[{"metadata":{},"cell_type":"markdown","source":"Cell for notes"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport os\nimport sys\nsys.path.append(os.path.abspath('../input/efficientnet/efficientnet-master/efficientnet-master/'))\nfrom efficientnet import EfficientNetB5\nfrom sklearn.utils import class_weight, shuffle\nfrom keras import models\nfrom keras.models import Model\nfrom keras.layers import Dense, Dropout, GlobalAveragePooling2D, Input\nfrom keras.optimizers import Adam, Optimizer\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping, Callback\nfrom sklearn.model_selection import train_test_split\nfrom keras.regularizers import l2\nfrom keras.utils import Sequence, to_categorical\nimport keras.backend as K\nfrom keras.legacy import interfaces\nimport tensorflow as tf\nfrom numpy.random import seed\nfrom sklearn.metrics import cohen_kappa_score, confusion_matrix\nfrom imgaug import augmenters as iaa\nseed(1)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport cv2\nimport zipfile","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Loading the Data and preprocessing the images\n\nI used images from the 2019 competetion as well as the previous competition from 2015 . To keep the classes somewhat balanced, I'm only using 700 images from each class from the 2015 dataset. I did preprocessed the images locally by resizing every image to $(456,456)$, which is EfficientNetB5's native image size.\n\n## Notes\n\n - I did not crop the blackspace out of the training images at any point of this competition. Per Tom Aindow's work, there is an unwanted correlation between the black space and labels in the 2015 dataset that is not present in the 2019 public test set. However, any attempt at cropping consistently led to lower LB scores and lower validation scores during training. For reference, the cropping method I tried was similar to the circle cropping method found here.
\n - For most of the competition, I kwas using an image size of $(224,224)$. Changing the image size to $(456,456)$ was my single biggest jump in LB score.
\n
"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"TRAIN_PATH_NEW = '../input/upload3/train_images/train_images/'\nTRAIN_PATH_OLD = '../input/upload3/train_2015/train_2015/'\nTEST_PATH = '../input/aptos2019-blindness-detection/test_images/'\n\ndf_old = pd.read_csv('../input/upload2/trainlabels_2015.csv')\ndf_old.columns = ['id_code', 'diagnosis']\ndf_old['id_code'] = TRAIN_PATH_OLD + df_old['id_code'] + '.jpeg'\n\ndf_new = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ndf_new['id_code'] = TRAIN_PATH_NEW + df_new['id_code'] + \".png\"\n\ndf = pd.concat([df_old, df_new], axis = 0)\ndf_old['diagnosis'] = df_old['diagnosis'].astype(str)\ndf_new['diagnosis'] = df_new['diagnosis'].astype(str)\n\ntrain, test = train_test_split(df_new, test_size = .2, random_state = 123, stratify = df_new['diagnosis'])\n\nbatch_size = 4\nimage_size = (456,456)\n\nfigax, ax = plt.subplots(1,1,figsize = (10,5))\nsns.barplot(x = df['diagnosis'].value_counts().index, y = df['diagnosis'].value_counts()).set_title(\"Diagnosis Distribution\")\nsns.despine(bottom = True, left = True)\nsns.set_style('dark')\nax.set_ylabel(\"\")\nax.get_yaxis().set_visible(False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring the images\n\nSince there is a large amount of variability in the images (e.g., brightness and contrast), we need to do some image preprocessing before training a model. Like many other participants in this competition, I used Ben Graham's preprocessing method described in this kernel. As mentioned above, I also considered cropping blackspace while preprocessing the images, but ultimately I decided against it. Huge shoutout to @ratthachat for giving so many of us a place to start in this competition."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize = (16,16))\n# Test image preprocessing function\nfor i in range(6):\n img = cv2.imread(df.iloc[i,:]['id_code'])\n img = cv2.resize(img, image_size)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n mask = gray > 10\n img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n \n crop = np.stack([img1,img2,img3],axis=-1)\n crop_transform = cv2.addWeighted (crop,4, cv2.GaussianBlur(crop , (0,0) ,10) ,-4 ,128)\n crop = cv2.resize(crop, image_size)\n crop_transform = cv2.resize(crop_transform, image_size)\n ax = fig.add_subplot(4,6,i+1)\n plt.imshow(img)\n ax2 = fig.add_subplot(4,6,i+7)\n plt.imshow(gray, cmap='gray')\n ax3 = fig.add_subplot(4,6,i+13)\n plt.imshow(crop)\n ax4 = fig.add_subplot(4,6,i+19)\n plt.imshow(crop_transform)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image generator and augmentations\n\n"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class ImgGenerator(Sequence):\n ########################################################################################\n ## Generic custom Generator.\n ## Parameters:\n ## df (req): Pandas DataFrame\n ## x_col (req): Column name with filenames\n ## y_col (req): Column with class labels\n ## file_path (opt): File Path\n ## img_extension (opt): image file extension (if not included in df[x_col])\n ## img_size (opt): Image size, defautls 320 x 320\n ## batch_size (opt): batch_size. defaults to 32\n ## should_shuffle (opt): If True, shuffles df after each epoch. Defaults True\n ## should_aug (opt): If True, augments images.\n ## preprocessing_function (opt): function to call on each image before augmentation\n ## postprocessing_function (opt): function to call on each image after augmentation\n ## iaa_seq (opt): augmentation sequence to run on each image\n ########################################################################################\n def __init__(self,\n df = None,\n x_col = None,\n y_col = None,\n num_classes = None,\n file_path = \"\",\n img_extension = \"\",\n img_size = (320,320),\n batch_size = 32,\n should_shuffle = True,\n should_aug = False,\n preprocessing_function = None,\n postprocessing_function = None,\n should_mix = True,\n iaa_seq = None):\n self.df = df\n self.x_col = x_col\n self.y_col = y_col\n self.num_classes = num_classes\n self.file_path = file_path\n self.img_extension = img_extension\n self.img_size = img_size\n self.batch_size = batch_size\n self.should_shuffle = should_shuffle\n self.should_aug = should_aug\n self.preprocessing_function = preprocessing_function\n self.postprocessing_function = postprocessing_function\n self.iaa_seq = iaa_seq\n self.should_mix = should_mix\n \n def __len__(self):\n return int(np.ceil(self.df.shape[0] / float(self.batch_size)))\n \n def __getitem__(self, idx):\n batch_x = self.df[self.x_col][idx * self.batch_size:(idx + 1) * self.batch_size]\n if self.y_col is not None:\n if self.num_classes is not None:\n batch_y = to_categorical(self.df[self.y_col][idx * self.batch_size:(idx + 1) * self.batch_size], num_classes = self.num_classes)\n else:\n batch_y = self.df[self.y_col][idx * self.batch_size:(idx + 1) * self.batch_size]\n else:\n batch_y = None\n return self.generate_images(batch_x, batch_y, self.should_aug)\n \n def on_epoch_end(self):\n if self.should_shuffle:\n self.df = self.df.sample(frac = 1)\n \n \n # https://arxiv.org/pdf/1710.09412.pdf\n def mix_up(self, x, y):\n lam = np.random.beta(0.2, 0.4)\n ori_index = np.arange(int(len(x)))\n index_array = np.arange(int(len(x)))\n np.random.shuffle(index_array) \n \n mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n \n return mixed_x, mixed_y\n \n def generate_images(self, batch_x, batch_y, should_aug):\n batch_images = []\n if batch_y is None:\n for (file) in batch_x:\n if self.img_extension != \"\":\n full_path = os.path.join(self.file_path, file + \".\" + self.img_extension)\n else:\n full_path = os.path.join(self.file_path, file)\n img = cv2.imread(full_path)\n img = cv2.resize(img, self.img_size)\n \n # Run preprocessing function\n if self.preprocessing_function is not None:\n img = self.preprocessing_function(img)\n \n # Run post processing function \n if self.postprocessing_function is not None:\n img = self.postprocessing_function(img)\n batch_images.append(img)\n batch_images = np.array(batch_images, np.float32) / 255\n return batch_images\n \n for (file, label) in zip(batch_x, batch_y):\n # Read and resize\n if self.img_extension != \"\":\n full_path = os.path.join(self.file_path, file + \".\" + self.img_extension)\n else:\n full_path = os.path.join(self.file_path, file)\n img = cv2.imread(full_path)\n \n \n # Run preprocessing function\n if self.preprocessing_function is not None:\n img = self.preprocessing_function(img)\n \n # Augment as needed\n if self.should_aug:\n img = self.iaa_seq.augment_image(img)\n \n # Run post processing function \n if self.postprocessing_function is not None:\n img = self.postprocessing_function(img)\n img = cv2.resize(img, self.img_size)\n batch_images.append(img)\n \n # Scale by default (not configurable)\n batch_images = np.array(batch_images, np.float32) / 255\n batch_y = np.array(batch_y, np.float32)\n if self.should_mix:\n batch_images, batch_y = self.mix_up(batch_images, batch_y)\n return batch_images, batch_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing function to crop blackspace before augmentation.\n# postprocess_input applies Ben's preprocessing function after augmentation\n# See: https://www.kaggle.com/ratthachat/aptos-updatedv14-preprocessing-ben-s-cropping\n\ndef preprocess_input(x):\n gray = cv2.cvtColor(x, cv2.COLOR_RGB2GRAY)\n mask = gray > 7\n if x[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0] > 0:\n img1=x[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n img2=x[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n img3=x[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n x = np.stack([img1,img2,img3],axis=-1)\n return x\n\ndef postprocess_input(x):\n return cv2.addWeighted(x,4, cv2.GaussianBlur( x , (0,0) ,10) ,-4 ,128)\n\n# Sequential image augmentor.\n# I used horizontal flips, Gaussian Blurring, Gaussian Noise, and affine transformations.\n\nseq = iaa.Sequential([\n iaa.Fliplr(0.5),\n iaa.Sometimes(0.5,iaa.GaussianBlur(sigma=(0, 0.5))),\n iaa.ContrastNormalization((0.9, 1.2)),\n iaa.Sometimes(0.5,iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5)),\n iaa.Sometimes(0.5,iaa.Multiply((0.8, 1.2), per_channel=0.2)),\n iaa.Affine(\n scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n rotate=(-120, 120),\n shear=(-15, 15)\n )\n], random_order=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"pretrain_generator = ImgGenerator(df = df_old,\n x_col = \"id_code\",\n y_col = 'diagnosis',\n batch_size = batch_size,\n #num_classes = 5,\n img_size = image_size,\n #preprocessing_function = preprocess_input,\n postprocessing_function = postprocess_input,\n should_aug = True,\n should_mix = False,\n iaa_seq = seq)\npretrain_validation_generator = ImgGenerator(df = df_new,\n x_col = \"id_code\",\n y_col = 'diagnosis',\n batch_size = batch_size,\n #num_classes = 5,\n img_size = image_size,\n #preprocessing_function = preprocess_input,\n postprocessing_function = postprocess_input,\n should_aug = False,\n should_mix = False)\n\ntrain_generator = ImgGenerator(df = train,\n x_col = \"id_code\",\n y_col = 'diagnosis',\n batch_size = batch_size,\n #num_classes = 5,\n img_size = image_size,\n #preprocessing_function = preprocess_input,\n postprocessing_function = postprocess_input,\n should_aug = True,\n should_mix = False,\n iaa_seq = seq)\n\nvalidation_generator = ImgGenerator(df = test,\n x_col = \"id_code\",\n y_col = 'diagnosis',\n batch_size = batch_size,\n #num_classes = 5,\n img_size = image_size,\n #preprocessing_function = preprocess_input,\n postprocessing_function = postprocess_input,\n should_aug = False,\n should_mix = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Accumulating Adam Optimizer\nSince the model we're using is memory intestive, we are unable to use a large batch size during training. This is problematic since small batch sizes can lead to overfitting. To handle this problem, we define an accumulating Adam Optimizer. This optimizer will process several batches before updating the weights of the model. Source"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class AdamAccumulate(Optimizer):\n\n def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):\n if accum_iters < 1:\n raise ValueError('accum_iters must be >= 1')\n super(AdamAccumulate, self).__init__(**kwargs)\n with K.name_scope(self.__class__.__name__):\n self.iterations = K.variable(0, dtype='int64', name='iterations')\n self.lr = K.variable(lr, name='lr')\n self.beta_1 = K.variable(beta_1, name='beta_1')\n self.beta_2 = K.variable(beta_2, name='beta_2')\n self.decay = K.variable(decay, name='decay')\n if epsilon is None:\n epsilon = K.epsilon()\n self.epsilon = epsilon\n self.initial_decay = decay\n self.amsgrad = amsgrad\n self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))\n self.accum_iters_float = K.cast(self.accum_iters, K.floatx())\n\n @interfaces.legacy_get_updates_support\n def get_updates(self, loss, params):\n grads = self.get_gradients(loss, params)\n self.updates = [K.update_add(self.iterations, 1)]\n\n lr = self.lr\n\n completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())\n\n if self.initial_decay > 0:\n lr = lr * (1. / (1. + self.decay * completed_updates))\n\n t = completed_updates + 1\n\n lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))\n\n # self.iterations incremented after processing a batch\n # batch: 1 2 3 4 5 6 7 8 9\n # self.iterations: 0 1 2 3 4 5 6 7 8\n # update_switch = 1: x x (if accum_iters=4) \n update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)\n update_switch = K.cast(update_switch, K.floatx())\n\n ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n\n if self.amsgrad:\n vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n else:\n vhats = [K.zeros(1) for _ in params]\n\n self.weights = [self.iterations] + ms + vs + vhats\n\n for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):\n\n sum_grad = tg + g\n avg_grad = sum_grad / self.accum_iters_float\n\n m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad\n v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)\n\n if self.amsgrad:\n vhat_t = K.maximum(vhat, v_t)\n p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)\n self.updates.append(K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t))\n else:\n p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)\n\n self.updates.append(K.update(m, (1 - update_switch) * m + update_switch * m_t))\n self.updates.append(K.update(v, (1 - update_switch) * v + update_switch * v_t))\n self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))\n new_p = p_t\n\n # Apply constraints.\n if getattr(p, 'constraint', None) is not None:\n new_p = p.constraint(new_p)\n\n self.updates.append(K.update(p, (1 - update_switch) * p + update_switch * new_p))\n return self.updates\n\n def get_config(self):\n config = {'lr': float(K.get_value(self.lr)),\n 'beta_1': float(K.get_value(self.beta_1)),\n 'beta_2': float(K.get_value(self.beta_2)),\n 'decay': float(K.get_value(self.decay)),\n 'epsilon': self.epsilon,\n 'amsgrad': self.amsgrad}\n base_config = super(AdamAccumulate, self).get_config()\n return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Custom Callback to evaluate quadratic weighted kappa\nThis competition is scored using quadratic weighted Kappa. It is not easy to implement quadratic weighted kappa as an evaluation metric. As a workaround, we calculate the quadratic weighted kappa after each epoch as a callback. This callback object also stores a history of quadratic weighted kappa, confusion matrices, and value_counts DataFrames for each epoch.\n\n## Notes:\n\n - Once I switched to a larger image size, I stopped using this callback since it increases the training time by about sixty seconds per epoch.
\n - Quadratic weighted kappa turned out to be volatile during training. It was much smoother to use mean squared error as a loss function.
"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class EvaluationMetrics(Callback):\n def __init__(self, validation_data = None, batch_size = 32):\n super(Callback, self).__init__()\n self.validation_generator = validation_data\n self.batch_size = batch_size\n \n def on_train_begin(self, logs={}):\n self._val_qwk = []\n self._val_pred_counts = pd.DataFrame(index = [0,1,2,3,4])\n self._val_confusion_matrices = []\n self._epoch = 1\n\n def on_epoch_end(self, batch, logs={}):\n y_val = np.array([])\n for i in range(len(validation_generator)):\n y_val_batch = validation_generator[i][1]\n y_val = np.append(y_val, y_val_batch)\n y_val.astype(int)\n y_predict = model.predict_generator(generator=validation_generator,\n steps=np.ceil(float(len(y_val)) / float(batch_size)),\n verbose = 1).tolist()\n\n for i in range(len(y_predict)):\n if y_predict[i][0] <= 0.5:\n y_predict[i][0] = 0\n elif y_predict[i][0] <= 1.5:\n y_predict[i][0] = 1\n elif y_predict[i][0] <= 2.5:\n y_predict[i][0] = 2\n elif y_predict[i][0] <= 3.5:\n y_predict[i][0] = 3\n else:\n y_predict[i][0] = 4\n \n y_predict = np.array(y_predict)\n \n self.qwk_hx(y_val, y_predict)\n self.value_counts_hx(y_predict)\n self.confusion_matrix_hx(y_val, y_predict)\n self._epoch += 1\n return\n \n def qwk_hx(self, y_val, y_predict):\n qwk = cohen_kappa_score(y_val,\n y_predict,\n labels = [0,1,2,3,4],\n weights='quadratic')\n print(\"\\nquadratic weighted kappa score: {0}\\n\".format(qwk))\n self._val_qwk.append(qwk)\n return\n \n def value_counts_hx(self, y_predict):\n self._val_pred_counts['Epoch {0}'.format(self._epoch)] = pd.DataFrame(y_predict)[0].value_counts()\n self._val_pred_counts.fillna(0,inplace=True)\n self._val_pred_counts = self._val_pred_counts.astype(int)\n print(\"Value counts at Epoch {0}:\\n\".format(self._epoch))\n print(self._val_pred_counts.head())\n return\n \n def confusion_matrix_hx(self, y_val, y_predict):\n self._val_confusion_matrices.append(confusion_matrix(y_val, y_predict))\n return\n\n def get_qwk_hx(self):\n return self._val_qwk\n \n def get_value_counts_hx(self):\n return self._val_pred_counts\n \n def get_confusion_matrix_hx(self):\n return self.val_confusion_matrices\n\neval_metrics = EvaluationMetrics(validation_data = validation_generator, batch_size = batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define and train the model\nThe model itself is fairly simple. I'm using EfficientNetB5 with weights pretrained on the imagenet dataset. The model was trained and validated using the 2019 images mixed with 3,500 images from the 2015 dataset (700 from each class)"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def mse (y_true, y_pred):\n return K.mean(K.square(y_pred -y_true), axis=-1)\n\ninputs = Input((456, 456, 3))\nbase_model = EfficientNetB5(include_top=False, weights=None, input_tensor = inputs)\nbase_model.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')\nout = GlobalAveragePooling2D()(base_model.output)\nout = Dropout(0.5)(out)\nout = Dense(1024,activation=\"relu\", kernel_regularizer=l2(0.01))(out)\nout = Dropout(0.5)(out)\nout = Dense(1,activation=\"relu\", kernel_regularizer=l2(0.01))(out)\nmodel = Model(inputs, out)\n# Define Callsbacks\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', \n patience=3, \n verbose=1, \n factor=0.5, \n min_lr=0.0000001)\n\nearly_stopping = EarlyStopping(monitor = 'val_loss',\n patience = 10,\n restore_best_weights = True)\n\n# Skipping the training in this kernel to speed up commit time.\n'''\n# Begin model warmup\nfor layer in model.layers[:-5]:\n layer.trainable = False\n \nmodel.compile(loss='mse', optimizer=AdamAccumulate(lr=0.001, accum_iters = 4), metrics=[mse])\n\nprint(\"\\nWarming up model: \" + \"\\n\"*2)\nepochs = 5\nhistory = model.fit_generator(pretrain_generator,\n epochs=epochs,\n steps_per_epoch = (df_old.shape[0] // batch_size) + 1,\n validation_data = pretrain_validation_generator,\n validation_steps = (df_new.shape[0] // batch_size) + 1,\n workers = 2,\n #callbacks = [eval_metrics],\n use_multiprocessing = True,\n verbose=1)\n\n# Train all layers\nfor layer in model.layers:\n layer.trainable = True\n\nprint('\\nBegining training session: ' + '\\n'*2)\nmodel.compile(loss='mse', optimizer=AdamAccumulate(lr=0.0005, accum_iters = 6), metrics=[mse])\n\nepochs = 10\nhistory2 = model.fit_generator(pretrain_generator,\n epochs=epochs,\n steps_per_epoch = (df_old.shape[0] // batch_size) + 1,\n validation_data = pretrain_validation_generator,\n validation_steps = (df_new.shape[0] // batch_size) + 1,\n callbacks=[learning_rate_reduction, early_stopping],# eval_metrics],\n workers = 2,\n use_multiprocessing = True,\n verbose=1)\n\nepochs = 40\nhistory3 = model.fit_generator(train_generator,\n epochs=epochs,\n steps_per_epoch = (train.shape[0] // batch_size) + 1,\n validation_data = validation_generator,\n validation_steps = (test.shape[0] // batch_size) + 1,\n callbacks=[learning_rate_reduction, early_stopping],# eval_metrics],\n workers = 2,\n use_multiprocessing = True,\n verbose=1)\nmodel.save('model.h5')\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Time Augmentation for final submission\n\nFor the final submission, I used 6-fold TTA using flips, affine transformations, and rotations of up to $90^\\circ$. The model predicted the class on the original image, as well as five augmented images. Then they voted on the label for the final submission. Note: in the event of a tie the model chose the lower label. This was not intentional, I forgot to fix it in time.\n"},{"metadata":{},"cell_type":"markdown","source":"## Evaluate predicted quadratic weighted kappa on validation set without TTA"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"y_val = np.array([])\nfor i in range(len(validation_generator)):\n v_val_batch = validation_generator[i][1]\n y_val = np.append(y_val, v_val_batch)\ny_val.astype(int)\ny_pred = model.predict_generator(generator=validation_generator,\n steps=np.ceil(float(len(y_val)) / float(batch_size)),\n verbose=1)\n\ny_pred[y_pred <= 0.5] = 0\ny_pred[(y_pred <= 1.5) & (y_pred > 0.5)] = 1\ny_pred[(y_pred <= 2.5) & (y_pred > 1.5)] = 2\ny_pred[(y_pred <= 3.5) & (y_pred > 2.5)] = 3\ny_pred[y_pred > 3.5] = 4\n\nscore = cohen_kappa_score(y_val,\n y_pred.astype(int),\n labels=[0,1,2,3,4],\n weights='quadratic')\n\nprint(\"Quadratic weighted kappa: {0}\".format(score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluate predicted quadratic weighted kappa on validation set with TTA"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"seq_tta = iaa.Sequential([\n iaa.Fliplr(0.5),\n iaa.Flipud(0.5),\n #iaa.Sometimes(0.5,iaa.GaussianBlur(sigma=(0, 0.5))),\n #iaa.ContrastNormalization((0.9, 1.2)),\n #iaa.Sometimes(0.5,iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5)),\n #iaa.Sometimes(0.5,iaa.Multiply((0.8, 1.2), per_channel=0.2)),\n iaa.Affine(\n scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n translate_percent={\"x\": (-0.05, 0.05), \"y\": (-0.05, 0.05)},\n rotate=(-90, 90),\n shear=(-15, 15)\n )\n], random_order=True)\n\ntta_df = test.copy()\n\npreds = []\nfor ix, idx in enumerate(tta_df['id_code']):\n image = cv2.imread(idx)\n image = cv2.resize(image, (456,456))\n score_ary = [0 for i in range(5)]\n for i in range(6):\n if i == 0:\n score_predict = model.predict((postprocess_input(image)[np.newaxis])/255)\n else:\n score_predict = model.predict((postprocess_input(seq_tta.augment_image(image))[np.newaxis])/255) \n if score_predict <= 0.5:\n score_predict = 0\n elif score_predict <= 1.5:\n score_predict = 1\n elif score_predict <= 2.5:\n score_predict = 2\n elif score_predict <= 3.5:\n score_predict = 3\n else:\n score_predict = 4\n score_ary[score_predict] += 1\n score_ary = np.array(score_ary)\n label = np.argmax(score_ary)\n preds.append(label) \n\ntta_df['diagnosis'] = np.array(preds, np.int)\ntta_df['id_code'] = tta_df['id_code'].str.split(\".png\", expand = True)[0]\n\nscore = cohen_kappa_score(test['diagnosis'].astype(int),\n list(tta_df['diagnosis']),\n labels=[0,1,2,3,4],\n weights='quadratic')\n \nprint(\"TTA score: {0}\".format(score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final submission with TTA"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sub_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nsub_df['id_code'] = sub_df['id_code']+'.png'\n\npreds = []\nfor ix, idx in enumerate(sub_df['id_code']):\n path = '../input/aptos2019-blindness-detection/test_images/' + idx\n image = cv2.imread(path)\n image = cv2.resize(image, (456,456))\n score_predict = [model.predict((postprocess_input(seq_tta.augment_image(image))[np.newaxis])/255) for i in range(5)]\n score_predict.append(model.predict((postprocess_input(image)[np.newaxis])/255))\n score_predict = sum(score_predict) / float(len(score_predict))\n if score_predict <= 0.5:\n score_predict = 0\n elif score_predict <= 1.5:\n score_predict = 1\n elif score_predict <= 2.5:\n score_predict = 2\n elif score_predict <= 3.5:\n score_predict = 3\n else:\n score_predict = 4\n preds.append(int(score_predict))\n\nsub_df['diagnosis'] = np.array(preds, np.int)\nsub_df['id_code'] = sub_df['id_code'].str.split(\".png\", expand = True)[0]\nsub_df.to_csv('submission.csv', index = False)\n\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"figax, ax = plt.subplots(1,1,figsize = (10,5))\nsns.barplot(x = sub_df['diagnosis'].value_counts().index, y = sub_df['diagnosis'].value_counts()).set_title(\"Predicted value distribution\")\nsns.despine(bottom = True, left = True)\nsns.set_style('dark')\nax.set_ylabel(\"\")\nax.get_yaxis().set_visible(False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize = (30,10))\nax = fig.add_subplot(1,2,1)\nsns.lineplot(x = range(1, len(history2.history['loss']) + 1), y = history2.history['loss'], ax = ax, label = 'Training Mean Squared Error')\nsns.lineplot(x = range(1, len(history2.history['val_loss']) + 1), y = history2.history['val_loss'], ax = ax, label = 'Validation Mean Squared Error')\nax.set_ylabel(\"\")\nax.set_yticks([0,0.5,1,1.5])\nax.set_xlabel(\"Epoch\")\nax.set_xticks(range(1,len(history2.history['val_loss']) + 1))\nax.set_title(\"Pre-Training History\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize = (30,10))\nax = fig.add_subplot(1,2,1)\nsns.lineplot(x = range(1, len(history3.history['loss']) + 1), y = history3.history['loss'], ax = ax, label = 'Training Mean Squared Error')\nsns.lineplot(x = range(1, len(history3.history['val_loss']) + 1), y = history3.history['val_loss'], ax = ax, label = 'Validation Mean Squared Error')\nax.set_ylabel(\"\")\nax.set_yticks([0, 0.25, 0.5, 0.75, 1])\nax.set_xlabel(\"Epoch\")\nax.set_xticks(range(1,len(history3.history['val_loss']) + 1))\nax.set_title(\"Training History\")\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}