{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"markdown","source":"import pandas as pd\nimport numpy as np\nimport cv2\nimport os\nimport matplotlib.pyplot as plt"},{"metadata":{"trusted":true},"cell_type":"code","source":"import multiprocessing\nnum_cores = multiprocessing.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/understanding_cloud_organization/train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_and_label = train_df[\"Image_Label\"].str.split(\"_\", expand = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"Image\"] = image_and_label[0]\ntrain_df[\"Label\"] = image_and_label[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir = \"/kaggle/input/understanding_cloud_organization/train_images/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hot one encoding cho moi hinh\n# Data augmentation??? => What types???\n# Architecture of the model\n# Training loop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr_df = pd.get_dummies(train_df, columns = ['Label'])\n# fill null values with '-1'\ncorr_df = corr_df.fillna('-1')\n\n# define a helper function to fill dummy columns\ndef get_dummy_value(row, cloud_type):\n    ''' Get value for dummy column '''\n    if cloud_type == 'fish':\n        return row['Label_Fish'] * (row['EncodedPixels'] != '-1')\n    if cloud_type == 'flower':\n        return row['Label_Flower'] * (row['EncodedPixels'] != '-1')\n    if cloud_type == 'gravel':\n        return row['Label_Gravel'] * (row['EncodedPixels'] != '-1')\n    if cloud_type == 'sugar':\n        return row['Label_Sugar'] * (row['EncodedPixels'] != '-1')\n    \n# fill dummy columns\ncorr_df['Label_Fish'] = corr_df.apply(lambda row: get_dummy_value(row, 'fish'), axis=1)\ncorr_df['Label_Flower'] = corr_df.apply(lambda row: get_dummy_value(row, 'flower'), axis=1)\ncorr_df['Label_Gravel'] = corr_df.apply(lambda row: get_dummy_value(row, 'gravel'), axis=1)\ncorr_df['Label_Sugar'] = corr_df.apply(lambda row: get_dummy_value(row, 'sugar'), axis=1)\n\n# check the result\ncorr_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = corr_df.groupby(\"Image\")['Label_Fish', 'Label_Flower', 'Label_Gravel', 'Label_Sugar'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"Image\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.iloc[:, 1:].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dictionary for fast access to ohe vectors\nimg_2_ohe_vector = {img:np.array(vec) for img, vec in zip(df['Image'], df.iloc[:, 1:].values)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_2_ohe_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np, os\nfrom PIL import Image \nimport cv2, keras, gc\nimport keras.backend as K\nfrom keras import layers\nfrom keras.models import Model\nfrom keras.models import load_model\nfrom keras.callbacks import LearningRateScheduler\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt, time\nfrom sklearn.metrics import roc_auc_score, accuracy_score\nimport random\nfrom sklearn.model_selection import train_test_split\nimport multiprocessing\nfrom copy import deepcopy\nfrom sklearn.metrics import precision_recall_curve, auc\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\nfrom keras.applications.densenet import DenseNet201\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image\nfrom tqdm import tqdm_notebook as tqdm\nfrom numpy.random import seed\nimport tensorflow as tf\nimport glob\nseed(10)\ntf.random.set_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nclass DataGenenerator(keras.utils.Sequence):\n    def __init__(self, images_list=None, folder_imgs=img_dir, \n                 batch_size=32, shuffle=True, augmentation=None,\n                 resized_height=260, resized_width=260, num_channels=3):\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augmentation = augmentation\n        if images_list is None:\n            self.images_list = os.listdir(folder_imgs)\n        else:\n            self.images_list = copy.deepcopy(np.array(images_list))\n        self.folder_imgs = folder_imgs\n        self.len = len(self.images_list) // self.batch_size\n        self.resized_height = resized_height\n        self.resized_width = resized_width\n        self.num_channels = num_channels\n        self.num_classes = 4\n        self.is_test = not 'train' in folder_imgs\n        if not shuffle and not self.is_test:\n            self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]\n\n    def __len__(self):\n        return self.len\n    \n    def on_epoch_start(self):\n        if self.shuffle:\n            random.shuffle(self.images_list)\n\n    def __getitem__(self, idx):\n        current_batch = self.images_list[idx * self.batch_size: (idx + 1) * self.batch_size]\n        X = np.empty((self.batch_size, self.resized_height, self.resized_width, self.num_channels))\n        y = np.empty((self.batch_size, self.num_classes))\n        print(X, y)\n        for i, image_name in enumerate(current_batch):\n            path = os.path.join(self.folder_imgs, image_name)\n            img = cv2.resize(cv2.imread(path), (self.resized_height, self.resized_width)).astype(np.float32)\n            if not self.augmentation is None:\n                augmented = self.augmentation(image=img)\n                img = augmented['image']\n            X[i, :, :, :] = img/255.0\n            if not self.is_test:\n                y[i, :] = img_2_ohe_vector[image_name]\n        return X, y\n\n    def get_labels(self):\n        if self.shuffle:\n            images_current = self.images_list[:self.len*self.batch_size]\n            labels = [img_2_ohe_vector[img] for img in images_current]\n        else:\n            labels = self.labels\n        return np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import Normalize\nalbumentations_train = Compose([\n    VerticalFlip(), HorizontalFlip(), Rotate(limit=20), GridDistortion(), Normalize()\n], p=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+https://github.com/qubvel/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.keras as efn \nfrom keras.layers import Dense\ndef get_model():\n    K.clear_session()\n    base_model =  efn.EfficientNetB2(weights='imagenet', include_top=False, pooling='avg', input_shape=(1400, 2100, 3))\n    for idx, layer in enumerate(base_model.layers):\n        if idx != len(base_model.layers) - 1:\n            layer.trainable = False\n    x = base_model.output\n    y_pred = Dense(4, activation='sigmoid')(x)\n    return Model(inputs=base_model.input, outputs=y_pred)\n\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\nfrom keras.applications.densenet import DenseNet201\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nclass PrAucCallback(Callback):\n    def __init__(self, data_generator, num_workers=-1, \n                 early_stopping_patience=5, \n                 plateau_patience=3, reduction_rate=0.5,\n                 stage='train', checkpoints_path='checkpoints/'):\n        super(Callback, self).__init__()\n        self.data_generator = data_generator\n        self.num_workers = num_workers\n        self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']\n        self.history = [[] for _ in range(len(self.class_names) + 1)] # to store per each class and also mean PR AUC\n        self.early_stopping_patience = early_stopping_patience\n        self.plateau_patience = plateau_patience\n        self.reduction_rate = reduction_rate\n        self.stage = stage\n        self.best_pr_auc = -float('inf')\n        if not os.path.exists(checkpoints_path):\n            os.makedirs(checkpoints_path)\n        self.checkpoints_path = checkpoints_path\n        \n    def compute_pr_auc(self, y_true, y_pred):\n        pr_auc_mean = 0\n        print(f\"\\n{'#'*30}\\n\")\n        for class_i in range(len(self.class_names)):\n            precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])\n            pr_auc = auc(recall, precision)\n            pr_auc_mean += pr_auc/len(self.class_names)\n            print(f\"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}\\n\")\n            self.history[class_i].append(pr_auc)        \n        print(f\"\\n{'#'*20}\\n PR AUC mean, {self.stage}: {pr_auc_mean:.3f}\\n{'#'*20}\\n\")\n        self.history[-1].append(pr_auc_mean)\n        return pr_auc_mean\n              \n    def is_patience_lost(self, patience):\n        if len(self.history[-1]) > patience:\n            best_performance = max(self.history[-1][-(patience + 1):-1])\n            return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]    \n              \n    def early_stopping_check(self, pr_auc_mean):\n        if self.is_patience_lost(self.early_stopping_patience):\n            self.model.stop_training = True    \n              \n    def model_checkpoint(self, pr_auc_mean, epoch):\n        if pr_auc_mean > self.best_pr_auc:\n            # remove previous checkpoints to save space\n            for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')):\n                os.remove(checkpoint)\n            self.best_pr_auc = pr_auc_mean\n            self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))              \n            print(f\"\\n{'#'*20}\\nSaved new checkpoint\\n{'#'*20}\\n\")\n              \n    def reduce_lr_on_plateau(self):\n        if self.is_patience_lost(self.plateau_patience):\n            new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate\n            keras.backend.set_value(self.model.optimizer.lr, new_lr)\n            print(f\"\\n{'#'*20}\\nReduced learning rate to {new_lr}.\\n{'#'*20}\\n\")\n        \n    def on_epoch_end(self, epoch, logs={}):\n        y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)\n        y_true = self.data_generator.get_labels()\n        # estimate AUC under precision recall curve for each class\n        pr_auc_mean = self.compute_pr_auc(y_true, y_pred)\n              \n        if self.stage == 'val':\n            # early stop after early_stopping_patience=4 epochs of no improvement in mean PR AUC\n            self.early_stopping_check(pr_auc_mean)\n\n            # save a model with the best PR AUC in validation\n            self.model_checkpoint(pr_auc_mean, epoch)\n\n            # reduce learning rate on PR AUC plateau\n            self.reduce_lr_on_plateau()            \n        \n    def get_pr_auc_history(self):\n        return self.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VAL = 0.2\ntrain_img_files = list(df[\"Image\"][:int((1-VAL) * len(df))])\nval_img_files = list(df[\"Image\"][int((1-VAL) * len(df)):])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_files\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_generator_train = DataGenenerator(train_img_files, img_dir, augmentation = albumentations_train)\ndata_generator_train_eval = DataGenenerator(train_img_files, shuffle=False)\ndata_generator_val = DataGenenerator(val_img_files, img_dir, augmentation = albumentations_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metric_callback = PrAucCallback(data_generator_train_eval)\nval_callback = PrAucCallback(data_generator_val, stage='val')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer= keras.optimizers.Adam(amsgrad=False,name=\"Adam\"),  loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory_0 = model.fit(x=data_generator_train,\n                              epochs=20,\n                              callbacks=[train_metric_callback, val_callback],\n                              workers=num_cores,\n                              verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"/kaggle/output/model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}