{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import os, glob\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport multiprocessing\nfrom copy import deepcopy\nfrom sklearn.metrics import precision_recall_curve, auc\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\nfrom keras.applications.densenet import DenseNet169\nfrom keras.applications.densenet import DenseNet121\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image\nfrom tqdm import tqdm_notebook as tqdm\nfrom numpy.random import seed\nseed(10)\nfrom tensorflow import set_random_seed\nset_random_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"test_imgs_folder = '../input/understanding_cloud_organization/test_images/'\ntrain_imgs_folder = '../input/understanding_cloud_organization/train_images/'\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Generators","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## One-hot encoding classes","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/understanding_cloud_organization/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"train_df = train_df[~train_df['EncodedPixels'].isnull()]\ntrain_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])\ntrain_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])\nclasses = train_df['Class'].unique()\ntrain_df = train_df.groupby('Image')['Class'].agg(set).reset_index()\nfor class_name in classes:\n    train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dictionary for fast access to ohe vectors\nimg_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"print(img_2_ohe_vector)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"print(train_df['Class'].map(lambda x: str(sorted(list(x)))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stratified split into train/val","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs, val_imgs = train_test_split(train_df['Image'].values, \n                                        test_size=0.2, \n                                        stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))), # sorting present classes in lexicographical order, just to be sure\n                                        random_state=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generator class","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenenerator(Sequence):\n    def __init__(self, images_list=None, folder_imgs=train_imgs_folder, \n                 batch_size=32, shuffle=True, augmentation=None,\n                 resized_height=224, resized_width=224, num_channels=3):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        if images_list is None:\n            self.images_list = os.listdir(folder_imgs)\n        else:\n            self.images_list = deepcopy(images_list)\n        self.folder_imgs = folder_imgs\n        self.len = len(self.images_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.num_classes = 4\n        self.is_test = not 'train' in folder_imgs\n        if not shuffle and not self.is_test:\n            self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.images_list)\n\n    def __getitem__(self, idx):\n        current_batch = self.images_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n        y = np.empty((self.batch_size, self.num_classes))\n\n        for i, image_name in enumerate(current_batch):\n            path = os.path.join(self.folder_imgs, image_name)\n            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img/255.0\n            if not self.is_test:\n                y[i, :] = img_2_ohe_vector[image_name]\n        return X, y\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.images_list[:self.len*self.batch_size]\n            labels = [img_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"albumentations_train = Compose([\n    VerticalFlip(), HorizontalFlip(), Rotate(limit=30), GridDistortion()\n], p=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generator instances","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)\ndata_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)\ndata_generator_val = DataGenenerator(val_imgs, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(data_generator_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PR-AUC-based Callback","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The callback would be used:\n1. to estimate AUC under precision recall curve for each class,\n2. to early stop after 5 epochs of no improvement in mean PR AUC,\n3. save a model with the best PR AUC in validation,\n4. to reduce learning rate on PR AUC plateau.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class PrAucCallback(Callback):\n    def __init__(self, data_generator, num_workers=num_cores, \n                 early_stopping_patience=5, \n                 plateau_patience=3, reduction_rate=0.2,\n                 stage='train', checkpoints_path='checkpoints/'):\n        super(Callback, self).__init__()\n        self.data_generator = data_generator\n        self.num_workers = num_workers\n        self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n        self.history = [[] for _ in range(len(self.class_names) + 1)] # to store per each class and also mean PR AUC\n        self.early_stopping_patience = early_stopping_patience\n        self.plateau_patience = plateau_patience\n        self.reduction_rate = reduction_rate\n        self.stage = stage\n        self.best_pr_auc = -float('inf')\n        if not os.path.exists(checkpoints_path):\n            os.makedirs(checkpoints_path)\n        self.checkpoints_path = checkpoints_path\n        \n    def compute_pr_auc(self, y_true, y_pred):\n        pr_auc_mean = 0\n        print(f\"\\n{'#'*30}\\n\")\n        for class_i in range(len(self.class_names)):\n            precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n            pr_auc = auc(recall, precision)\n            pr_auc_mean += pr_auc/len(self.class_names)\n            print(f\"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}\\n\")\n            self.history[class_i].append(pr_auc)        \n        print(f\"\\n{'#'*20}\\n PR AUC mean, {self.stage}: {pr_auc_mean:.3f}\\n{'#'*20}\\n\")\n        self.history[-1].append(pr_auc_mean)\n        return pr_auc_mean\n              \n    def is_patience_lost(self, patience):\n        if len(self.history[-1]) > patience:\n            best_performance = max(self.history[-1][-(patience + 1):-1])\n            return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]    \n              \n    def early_stopping_check(self, pr_auc_mean):\n        if self.is_patience_lost(self.early_stopping_patience):\n            self.model.stop_training = True    \n              \n    def model_checkpoint(self, pr_auc_mean, epoch):\n        if pr_auc_mean > self.best_pr_auc:\n            # remove previous checkpoints to save space\n            for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')):\n                os.remove(checkpoint)\n            self.best_pr_auc = pr_auc_mean\n            self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))              \n            print(f\"\\n{'#'*20}\\nSaved new checkpoint\\n{'#'*20}\\n\")\n              \n    def reduce_lr_on_plateau(self):\n        if self.is_patience_lost(self.plateau_patience):\n            new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate\n            keras.backend.set_value(self.model.optimizer.lr, new_lr)\n            print(f\"\\n{'#'*20}\\nReduced learning rate to {new_lr}.\\n{'#'*20}\\n\")\n        \n    def on_epoch_end(self, epoch, logs={}):\n        y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)\n        y_true = self.data_generator.get_labels()\n        # estimate AUC under precision recall curve for each class\n        pr_auc_mean = self.compute_pr_auc(y_true, y_pred)\n              \n        if self.stage == 'val':\n            # early stop after early_stopping_patience=4 epochs of no improvement in mean PR AUC\n            self.early_stopping_check(pr_auc_mean)\n\n            # save a model with the best PR AUC in validation\n            self.model_checkpoint(pr_auc_mean, epoch)\n\n            # reduce learning rate on PR AUC plateau\n            self.reduce_lr_on_plateau()            \n        \n    def get_pr_auc_history(self):\n        return self.history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Callback instances","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metric_callback = PrAucCallback(data_generator_train_eval)\nval_callback = PrAucCallback(data_generator_val, stage='val')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Classifier","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Defining a model","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install -U git+https://github.com/qubvel/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import efficientnet.keras as efn\ndef get_model():\n    K.clear_session()\n    base_model =  efn.EfficientNetB2(weights='imagenet', include_top=False, pooling='avg', input_shape=(224, 224, 3))\n    x = base_model.output\n    y_pred = Dense(4, activation='sigmoid')(x)\n    return Model(inputs=base_model.input, outputs=y_pred)\n\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial tuning of the added fully-connected layer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for base_layer in model.layers[:-1]:\n    base_layer.trainable = False\n    \nmodel.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy')\nhistory_0 = model.fit_generator(generator=data_generator_train,\n                              validation_data=data_generator_val,\n                              epochs=20,\n                              callbacks=[train_metric_callback, val_callback],\n                              workers=num_cores,\n                              verbose=1\n                             )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fine-tuning the whole model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"After unfreezing all the layers I set a less aggressive initial learning rate and train until early stopping (or 100 epochs max).","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"for base_layer in model.layers[:-1]:\n    base_layer.trainable = True\n    \nmodel.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy')\nhistory_1 = model.fit_generator(generator=data_generator_train,\n                              validation_data=data_generator_val,\n                              epochs=30,\n                              callbacks=[train_metric_callback, val_callback],\n                              workers=num_cores,\n                              verbose=1,\n                              initial_epoch=1\n                             )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing train and val PR AUC","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def plot_with_dots(ax, np_array):\n    ax.scatter(list(range(1, len(np_array) + 1)), np_array, s=50)\n    ax.plot(list(range(1, len(np_array) + 1)), np_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pr_auc_history_train = train_metric_callback.get_pr_auc_history()\npr_auc_history_val = val_callback.get_pr_auc_history()\n\nplt.figure(figsize=(10, 7))\nplot_with_dots(plt, pr_auc_history_train[-1])\nplot_with_dots(plt, pr_auc_history_val[-1])\n\nplt.xlabel('Epoch', fontsize=15)\nplt.ylabel('Mean PR AUC', fontsize=15)\nplt.legend(['Train', 'Val'])\nplt.title('Training and Validation PR AUC', fontsize=20)\nplt.savefig('pr_auc_hist.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nplot_with_dots(plt, history_0.history['loss']+history_1.history['loss'])\nplot_with_dots(plt, history_0.history['val_loss']+history_1.history['val_loss'])\n\nplt.xlabel('Epoch', fontsize=15)\nplt.ylabel('Binary Crossentropy', fontsize=15)\nplt.legend(['Train', 'Val'])\nplt.title('Training and Validation Loss', fontsize=20)\nplt.savefig('loss_hist.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I left the model to train longer on my local GPU. I then upload the best model and plots from the model training.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = load_model('../input/clouds-classifier-files/classifier_densenet169_epoch_21_val_pr_auc_0.8365921057512743.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Selecting postprocessing thresholds","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n# def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.95, precision_threshold=0.94, plot=False):\n#     precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n#     i = len(thresholds) - 1\n#     best_recall_threshold = None\n#     while best_recall_threshold is None:\n#         next_threshold = thresholds[i]\n#         next_recall = recall[i]\n#         if next_recall >= recall_threshold:\n#             best_recall_threshold = next_threshold\n#         i -= 1\n        \n#     # consice, even though unnecessary passing through all the values\n#     best_precision_threshold = [thres for prec, thres in zip(precision, thresholds) if prec >= precision_threshold][0]\n    \n#     if plot:\n#         plt.figure(figsize=(10, 7))\n#         plt.step(recall, precision, color='r', alpha=0.3, where='post')\n#         plt.fill_between(recall, precision, alpha=0.3, color='r')\n#         plt.axhline(y=precision[i + 1])\n#         recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds) \n#                                  if thres == best_precision_threshold][0]\n#         plt.axvline(x=recall_for_prec_thres, color='g')\n#         plt.xlabel('Recall')\n#         plt.ylabel('Precision')\n#         plt.ylim([0.0, 1.05])\n#         plt.xlim([0.0, 1.0])\n#         plt.legend(['PR curve', \n#                     f'Precision {precision[i + 1]: .2f} corresponding to selected recall threshold',\n#                     f'Recall {recall_for_prec_thres: .2f} corresponding to selected precision threshold'])\n#         plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')\n#     return best_recall_threshold, best_precision_threshold\n\n# y_pred = model.predict_generator(data_generator_val, workers=num_cores)\n# y_true = data_generator_val.get_labels()\n# recall_thresholds = dict()\n# precision_thresholds = dict()\n# for i, class_name in tqdm(enumerate(class_names)):\n#     recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# print(y_pred)#val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# print(y_true)#val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Post-processing segmentation submission","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Predicting cloud classes for test.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)\n# y_pred_test = model.predict_generator(data_generator_test, workers=num_cores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(y_pred_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Estimating set of images without masks.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# image_labels_empty = set()\n# for i, (img, predictions) in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)):\n#     #print(img,predictions)#predictions=y_pred_test\n#     for class_i, class_name in enumerate(class_names):#0 1 2 3 Fish Flower Sugar Gravel\n#         if predictions[class_i] < recall_thresholds[class_name]:\n#             image_labels_empty.add(f'{img}_{class_name}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Segmentation results:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.read_csv('../input/efficient-net-b4-unet-clouds/submission.csv')\n# submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull(), 'Image_Label'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(f'{len(image_labels_empty.intersection(predictions_nonempty))} masks would be removed')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #removing masks\n# submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan\n# submission.to_csv('submission_segmentation_and_classifier.csv', index=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}