{"cells":[{"metadata":{},"cell_type":"markdown","source":"I've used quite a few things from other kernels of the competetion to get staretd, just changed them so that I can understand them better."},{"metadata":{},"cell_type":"markdown","source":"# Install libraries"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install segmentation-models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport collections\nimport time \nimport sys\nimport tqdm\nfrom multiprocessing import  Pool\nfrom PIL import Image\nfrom functools import partial\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ntrain_on_gpu = True\n\n# Visualisation libs\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\n\nfrom albumentations import (\n    PadIfNeeded,\n    HorizontalFlip,\n    VerticalFlip,    \n    CenterCrop,    \n    Crop,\n    Compose,\n    Transpose,\n    RandomRotate90,\n    ElasticTransform,\n    GridDistortion, \n    OpticalDistortion,\n    RandomSizedCrop,\n    OneOf,\n    CLAHE,\n    RandomBrightnessContrast,    \n    RandomGamma    \n)\nimport segmentation_models as sm\n\nimport keras\nfrom keras import optimizers\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, Concatenate","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions and constants"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# used from notebooks mentioned in the end\ndef np_resize(img, input_shape):\n    height, width = input_shape\n    return cv2.resize(img, (width, height))\n    \ndef mask2rle(img):\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle2mask(rle, input_shape):\n    width, height = input_shape[:2]\n    \n    mask = np.zeros( width*height ).astype(np.uint8)\n    \n    array = np.asarray([int(x) for x in rle.split()])\n    starts = array[0::2]\n    lengths = array[1::2]\n\n    current_position = 0\n    for index, start in enumerate(starts):\n        mask[int(start):int(start+lengths[index])] = 1\n        current_position += lengths[index]\n        \n    return mask.reshape(height, width).T\n\ndef build_masks(rles, input_shape, reshape=None):\n    depth = len(rles)\n    if reshape is None:\n        masks = np.zeros((*input_shape, depth))\n    else:\n        masks = np.zeros((*reshape, depth))\n    \n    for i, rle in enumerate(rles):\n        if type(rle) is str:\n            if reshape is None:\n                masks[:, :, i] = rle2mask(rle, input_shape)\n            else:\n                mask = rle2mask(rle, input_shape)\n                reshaped_mask = np_resize(mask, reshape)\n                masks[:, :, i] = reshaped_mask\n    \n    return masks\n\ndef build_rles(masks, reshape=None):\n    width, height, depth = masks.shape\n    rles = []\n    \n    for i in range(depth):\n        mask = masks[:, :, i]\n        \n        if reshape:\n            mask = mask.astype(np.float32)\n            mask = np_resize(mask, reshape).astype(np.int64)\n        \n        rle = mask2rle(mask)\n        rles.append(rle)\n        \n    return rles\n\ndef dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Parameters\nLEARNING_RATE = 2e-3\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 4\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5\nBACKBONE = 'resnet50'\nBATCH_SIZE=16\nSEED=42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data and Basic analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/understanding_cloud_organization'\nos.listdir(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{path}/train.csv')\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(f'{path}/sample_submission.csv')\nsample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have almost 10k+ null `EncodedPixels` in data. We'll add it as a flag in new column later on"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`Image_Label` has both our label and image name, we will break them into two columns for further analysis in next steps"},{"metadata":{},"cell_type":"markdown","source":"## Add new columns"},{"metadata":{},"cell_type":"markdown","source":"We will add 3 new columns: name, label and has_null_encoded_pxs."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def parallelize_dataframe(df, func, n_cores=4):\n    df_split = np.array_split(df, n_cores)\n    pool = Pool(n_cores)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n\ndef add_features(df):\n    df['name'] = df['Image_Label'].apply(lambda x: x.split('_')[0])\n    df['label'] = df['Image_Label'].apply(lambda x: x.split('_')[1])\n    df['has_null_encoded_pxs'] = df['EncodedPixels'].isnull()\n    return df\n\ntrain = parallelize_dataframe(train, add_features)\n\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA and Augmentation"},{"metadata":{},"cell_type":"markdown","source":"Let's get the exact count of null and non null EncodedPixels rows"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(14, 6))\nax = sns.countplot(y=\"has_null_encoded_pxs\", data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also, remember, we've image with multiple mask. So, also check how many images have multiple masks.\nWe will see this by seeing mask count groupings for images."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train[ train.has_null_encoded_pxs == False ].groupby('name').size().value_counts().reset_index().rename(columns={ 'index': 'Number of masks', '0': 'Image Count'})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training model"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"mask_count_df = train.groupby('name').agg(np.sum).reset_index()\nmask_count_df.sort_values('has_null_encoded_pxs', ascending=False, inplace=True)\ntrain_idx, val_idx = train_test_split(mask_count_df.index, test_size=0.2, random_state=42)\nprint(train_idx.shape, val_idx.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, df, target_df=None, mode='fit',\n                 base_path=f'{path}/train_images',\n                 batch_size=BATCH_SIZE, dim=(1400, 2100), n_channels=CHANNELS, reshape=(HEIGHT, WIDTH), \n                 n_classes=N_CLASSES, random_state=SEED, shuffle=True):\n        self.dim = dim\n        self.batch_size = batch_size\n        self.df = df\n        self.mode = mode\n        self.base_path = base_path\n        self.target_df = target_df\n        self.list_IDs = list_IDs\n        self.reshape = reshape\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.random_state = random_state\n        \n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        list_IDs_batch = [self.list_IDs[k] for k in indexes]\n        \n        X = self.__generate_X(list_IDs_batch)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(list_IDs_batch)\n            \n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n\n        else:\n            raise AttributeError('The mode parameter should be set to \"fit\" or \"predict\".')\n        \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.seed(self.random_state)\n            np.random.shuffle(self.indexes)\n    \n    def __generate_X(self, list_IDs_batch):\n        'Generates data containing batch_size samples'\n        # Initialization\n        if self.reshape is None:\n            X = np.empty((self.batch_size, *self.dim, self.n_channels))\n        else:\n            X = np.empty((self.batch_size, *self.reshape, self.n_channels))\n        \n        # Generate data\n        for i, ID in enumerate(list_IDs_batch):\n            im_name = self.df['name'].iloc[ID]\n            img_path = f\"{self.base_path}/{im_name}\"\n            img = cv2.imread(img_path)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            img = img.astype(np.float32) / 255.\n            \n            if self.reshape is not None:\n                img = np_resize(img, self.reshape)\n            \n            # Store samples\n            X[i,] = img\n\n        return X\n    \n    def __generate_y(self, list_IDs_batch):\n        if self.reshape is None:\n            y = np.empty((self.batch_size, *self.dim, self.n_classes), dtype=int)\n        else:\n            y = np.empty((self.batch_size, *self.reshape, self.n_classes), dtype=int)\n        \n        for i, ID in enumerate(list_IDs_batch):\n            im_name = self.df['name'].iloc[ID]\n            image_df = self.target_df[self.target_df['name'] == im_name]\n            \n            rles = image_df['EncodedPixels'].values\n            \n            if self.reshape is not None:\n                masks = build_masks(rles, input_shape=self.dim, reshape=self.reshape)\n            else:\n                masks = build_masks(rles, input_shape=self.dim)\n            \n            y[i, ] = masks\n\n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"train_generator = DataGenerator(train_idx, df=mask_count_df, target_df=train)\n \nvalid_generator = DataGenerator(val_idx, df=mask_count_df, target_df=train)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"model = sm.Unet(\n           encoder_name=BACKBONE, \n           classes=N_CLASSES,\n           activation='sigmoid',\n           input_shape=(HEIGHT, WIDTH, CHANNELS))\n\nmodel.compile(optimizer=optimizers.Adam(lr=LEARNING_RATE), loss=binary_crossentropy, metrics=[dice_coef])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# using it from dimitreoliveira's kernel\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using `fit_generator`\n\nNote: takes 10-13 minutes per epoch"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"history = model.fit_generator(generator=train_generator,\n                              validation_data=valid_generator,\n                              epochs=20,\n                              callbacks=[es, rlrop],\n                              verbose=2).history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, sharex='col', figsize=(20, 12))\n\nax.plot(history['dice_coef'], label='Train Dice coefficient')\nax.plot(history['val_dice_coef'], label='Validation Dice coefficient')\nax.legend(loc='best')\nax.set_title('Dice coefficient')\n\nplt.xlabel('Epochs')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Predict test images masks"},{"metadata":{},"cell_type":"markdown","source":"Add same columns again in sample submission data. "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sample_submission = parallelize_dataframe(sample_submission, add_features)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create a new dataframe for unique images only."},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.DataFrame(sample_submission['name'].unique(), columns=['name'])\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"test_df = []\n\nfor i in range(0, test.shape[0], 500):\n    batch_idx = list(range(i, min(test.shape[0], i + 500)))\n    test_generator = DataGenerator(\n                     batch_idx,\n                     df=test,\n                     target_df=sample_submission,\n                     batch_size=1,\n                     reshape=(HEIGHT, WIDTH),\n                     dim=(350, 525),\n                     n_channels=CHANNELS,\n                     n_classes=N_CLASSES,\n                     random_state=SEED,\n                     base_path=f'{path}/test_images',\n                     mode='predict',\n                     shuffle=False)\n\n    batch_pred_masks = model.predict_generator(test_generator)\n\n    for j, b in enumerate(batch_idx):\n        filename = test['name'].iloc[b]\n        image_df = sample_submission[sample_submission['name'] == filename].copy()\n\n        pred_masks = batch_pred_masks[j, ].round().astype(int)\n        pred_rles = build_rles(pred_masks, reshape=(350, 525))\n        image_df['EncodedPixels'] = pred_rles\n        test_df.append(image_df)\n        \nsubmission_ready = pd.concat(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"submission_ready.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Review Submission"},{"metadata":{},"cell_type":"markdown","source":"TBD"},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"submission = submission_ready[['Image_Label', 'EncodedPixels']]\nsubmission.to_csv('submission.csv', index=False)\nprint('Added submission file')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install Kaggle\n# !kaggle competitions submit -c understanding_cloud_organization -f submission.csv -m \"Message\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Code and notebook references"},{"metadata":{},"cell_type":"markdown","source":"I've looked at lot of different computer vision and segmentation notebooks for this challange to get started.\nFor this compettion, I looked at notebooks of:\n1. [Artgor](https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools)\n2. [dimitreoliveira](https://www.kaggle.com/dimitreoliveira/understanding-clouds-eda-and-keras-u-net)\n\nReally thankful to both and others to share their work with us.\n\nAlso, for augumentation, I looked at official [sample notebook](https://github.com/albu/albumentations/blob/master/notebooks/example_kaggle_salt.ipynb).\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}