{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, glob\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport multiprocessing\nfrom copy import deepcopy\nfrom sklearn.metrics import precision_recall_curve, auc\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\nfrom keras.applications.densenet import DenseNet201\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image\nfrom tqdm import tqdm_notebook as tqdm\nfrom numpy.random import seed\nseed(10)\n#from tensorflow import set_random_seed\n#set_random_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_imgs_folder = '../input/understanding_cloud_organization/test_images/'\ntrain_imgs_folder = '../input/understanding_cloud_organization/train_images/'\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/understanding_cloud_organization/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[~train_df['EncodedPixels'].isnull()]\ntrain_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])\ntrain_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])\nclasses = train_df['Class'].unique()\ntrain_df = train_df.groupby('Image')['Class'].agg(set).reset_index()\nfor class_name in classes:\n    train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dictionary for fast access to ohe vectors\nimg_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs, val_imgs = train_test_split(train_df['Image'].values, \n                                        test_size=0.2, \n                                        stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))), # sorting present classes in lexicographical order, just to be sure\n                                        random_state=2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenenerator(Sequence):\n    def __init__(self, images_list=None, folder_imgs=train_imgs_folder, \n                 batch_size=32, shuffle=True, augmentation=None,\n                 resized_height=260, resized_width=260, num_channels=3):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        if images_list is None:\n            self.images_list = os.listdir(folder_imgs)\n        else:\n            self.images_list = deepcopy(images_list)\n        self.folder_imgs = folder_imgs\n        self.len = len(self.images_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.num_classes = 4\n        self.is_test = not 'train' in folder_imgs\n        if not shuffle and not self.is_test:\n            self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.images_list)\n\n    def __getitem__(self, idx):\n        current_batch = self.images_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n        y = np.empty((self.batch_size, self.num_classes))\n\n        for i, image_name in enumerate(current_batch):\n            path = os.path.join(self.folder_imgs, image_name)\n            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img/255.0\n            if not self.is_test:\n                y[i, :] = img_2_ohe_vector[image_name]\n        return X, y\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.images_list[:self.len*self.batch_size]\n            labels = [img_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"albumentations_train = Compose([\n    VerticalFlip(), HorizontalFlip(), Rotate(limit=20), GridDistortion()\n], p=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)\ndata_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)\ndata_generator_val = DataGenenerator(val_imgs, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PrAucCallback(Callback):\n    def __init__(self, data_generator, num_workers=num_cores, \n                 early_stopping_patience=5, \n                 plateau_patience=3, reduction_rate=0.5,\n                 stage='train', checkpoints_path='checkpoints/'):\n        super(Callback, self).__init__()\n        self.data_generator = data_generator\n        self.num_workers = num_workers\n        self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n        self.history = [[] for _ in range(len(self.class_names) + 1)] # to store per each class and also mean PR AUC\n        self.early_stopping_patience = early_stopping_patience\n        self.plateau_patience = plateau_patience\n        self.reduction_rate = reduction_rate\n        self.stage = stage\n        self.best_pr_auc = -float('inf')\n        if not os.path.exists(checkpoints_path):\n            os.makedirs(checkpoints_path)\n        self.checkpoints_path = checkpoints_path\n        \n    def compute_pr_auc(self, y_true, y_pred):\n        pr_auc_mean = 0\n        print(f\"\\n{'#'*30}\\n\")\n        for class_i in range(len(self.class_names)):\n            precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n            pr_auc = auc(recall, precision)\n            pr_auc_mean += pr_auc/len(self.class_names)\n            print(f\"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}\\n\")\n            self.history[class_i].append(pr_auc)        \n        print(f\"\\n{'#'*20}\\n PR AUC mean, {self.stage}: {pr_auc_mean:.3f}\\n{'#'*20}\\n\")\n        self.history[-1].append(pr_auc_mean)\n        return pr_auc_mean\n              \n    def is_patience_lost(self, patience):\n        if len(self.history[-1]) > patience:\n            best_performance = max(self.history[-1][-(patience + 1):-1])\n            return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]    \n              \n    def early_stopping_check(self, pr_auc_mean):\n        if self.is_patience_lost(self.early_stopping_patience):\n            self.model.stop_training = True    \n              \n    def model_checkpoint(self, pr_auc_mean, epoch):\n        if pr_auc_mean > self.best_pr_auc:\n            # remove previous checkpoints to save space\n            for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')):\n                os.remove(checkpoint)\n            self.best_pr_auc = pr_auc_mean\n            self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))              \n            print(f\"\\n{'#'*20}\\nSaved new checkpoint\\n{'#'*20}\\n\")\n              \n    def reduce_lr_on_plateau(self):\n        if self.is_patience_lost(self.plateau_patience):\n            new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate\n            keras.backend.set_value(self.model.optimizer.lr, new_lr)\n            print(f\"\\n{'#'*20}\\nReduced learning rate to {new_lr}.\\n{'#'*20}\\n\")\n        \n    def on_epoch_end(self, epoch, logs={}):\n        y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)\n        y_true = self.data_generator.get_labels()\n        # estimate AUC under precision recall curve for each class\n        pr_auc_mean = self.compute_pr_auc(y_true, y_pred)\n              \n        if self.stage == 'val':\n            # early stop after early_stopping_patience=4 epochs of no improvement in mean PR AUC\n            self.early_stopping_check(pr_auc_mean)\n\n            # save a model with the best PR AUC in validation\n            self.model_checkpoint(pr_auc_mean, epoch)\n\n            # reduce learning rate on PR AUC plateau\n            self.reduce_lr_on_plateau()            \n        \n    def get_pr_auc_history(self):\n        return self.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metric_callback = PrAucCallback(data_generator_train_eval)\nval_callback = PrAucCallback(data_generator_val, stage='val')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.losses import binary_crossentropy\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred):\n    smooth = 1.\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = y_true_f * y_pred_f\n    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return 1. - score\n\ndef bce_dice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+https://github.com/qubvel/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten\nfrom keras.layers import Activation, Dense\n\nmodel = Sequential()\nmodel.add(Conv2D(1024, kernel_size=3,activation='relu',input_shape=(260,260,3)))\nmodel.add(Conv2D(512, kernel_size=3,activation='relu'))\nmodel.add(Conv2D(256, kernel_size=3,activation='relu'))\nmodel.add(Conv2D(128, kernel_size=3,activation='relu'))\nmodel.add(Conv2D(64, kernel_size=3,activation='relu'))\nmodel.add(Conv2D(32, kernel_size=3,activation='relu'))\nmodel.add(Conv2D(16, kernel_size=3,activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(4, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam',  loss='categorical_crossentropy', metrics=['accuracy'])\nhistory_0 = model.fit_generator(generator=data_generator_train,\n                              validation_data=data_generator_val,\n                              epochs=20,\n                              callbacks=[train_metric_callback, val_callback],\n                              workers=num_cores,\n                              verbose=1\n                             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_with_dots(ax, np_array):\n    ax.scatter(list(range(1, len(np_array) + 1)), np_array, s=50)\n    ax.plot(list(range(1, len(np_array) + 1)), np_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pr_auc_history_train = train_metric_callback.get_pr_auc_history()\npr_auc_history_val = val_callback.get_pr_auc_history()\n\nplt.figure(figsize=(10, 7))\nplot_with_dots(plt, pr_auc_history_train[-1])\nplot_with_dots(plt, pr_auc_history_val[-1])\n\nplt.xlabel('Epoch', fontsize=15)\nplt.ylabel('Mean PR AUC', fontsize=15)\nplt.legend(['Train', 'Val'])\nplt.title('Training and Validation PR AUC', fontsize=20)\nplt.savefig('pr_auc_hist.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nplot_with_dots(plt, history_0.history['loss']+history_1.history['loss'])\nplot_with_dots(plt, history_0.history['val_loss']+history_1.history['val_loss'])\n\nplt.xlabel('Epoch', fontsize=15)\nplt.ylabel('Binary Crossentropy', fontsize=15)\nplt.legend(['Train', 'Val'])\nplt.title('Training and Validation Loss', fontsize=20)\nplt.savefig('loss_hist.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image(\"../input/clouds-classifier-files/loss_hist_densenet169.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Image(\"../input/clouds-classifier-files/pr_auc_hist_densenet169.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\ndef get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.94, precision_threshold=0.90, plot=False):\n    precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n    i = len(thresholds) - 1\n    best_recall_threshold = None\n    while best_recall_threshold is None:\n        next_threshold = thresholds[i]\n        next_recall = recall[i]\n        if next_recall >= recall_threshold:\n            best_recall_threshold = next_threshold\n        i -= 1\n        \n    # consice, even though unnecessary passing through all the values\n    best_precision_threshold = [thres for prec, thres in zip(precision, thresholds) if prec >= precision_threshold][0]\n    \n    if plot:\n        plt.figure(figsize=(10, 7))\n        plt.step(recall, precision, color='r', alpha=0.3, where='post')\n        plt.fill_between(recall, precision, alpha=0.3, color='r')\n        plt.axhline(y=precision[i + 1])\n        recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds) \n                                 if thres == best_precision_threshold][0]\n        plt.axvline(x=recall_for_prec_thres, color='g')\n        plt.xlabel('Recall')\n        plt.ylabel('Precision')\n        plt.ylim([0.0, 1.05])\n        plt.xlim([0.0, 1.0])\n        plt.legend(['PR curve', \n                    f'Precision {precision[i + 1]: .2f} corresponding to selected recall threshold',\n                    f'Recall {recall_for_prec_thres: .2f} corresponding to selected precision threshold'])\n        plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')\n    return best_recall_threshold, best_precision_threshold\n\ny_pred = model.predict_generator(data_generator_val, workers=num_cores)\ny_true = data_generator_val.get_labels()\nrecall_thresholds = dict()\nprecision_thresholds = dict()\nfor i, class_name in tqdm(enumerate(class_names)):\n    recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)\ny_pred_test = model.predict_generator(data_generator_test, workers=num_cores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_labels_empty = set()\nfor i, (img, predictions) in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)):\n    for class_i, class_name in enumerate(class_names):\n        if predictions[class_i] < recall_thresholds[class_name]:\n            image_labels_empty.add(f'{img}_{class_name}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/densenet201cloudy/densenet201.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull(), 'Image_Label'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'{len(image_labels_empty.intersection(predictions_nonempty))} masks would be removed')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing masks\nsubmission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan\nsubmission.to_csv('submission_segmentation_and_classifier.csv', index=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}