{"cells":[{"metadata":{"trusted":true,"_uuid":"5bf61e64c3561a577afb09faa8764f1bc5560bc0","_kg_hide-output":true},"cell_type":"markdown","source":"**1. Python imports**"},{"metadata":{"trusted":true,"_uuid":"2b1935c982cd366486c9f0e4c4350339a22e2891"},"cell_type":"code","source":"\"\"\"Python file path, image, and data processing libraries.\"\"\"\nimport random\nimport os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\"\"\"Deep learning libraries.\"\"\"\nimport tensorflow as tf\nimport keras\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, Adagrad, Adadelta\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, LearningRateScheduler, CSVLogger\nfrom keras.models import Model, Sequential, load_model, model_from_json\nfrom keras.layers import Flatten, Dense, Activation, Input, Dropout, Activation, BatchNormalization, Reshape\nfrom keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D\nfrom keras.regularizers import l2\nfrom keras.layers import Dense, Conv2D, BatchNormalization, Activation\nfrom keras.layers import AveragePooling2D, Input, Flatten\nfrom keras.optimizers import Adam\nfrom imgaug import augmenters as iaa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11b6fbca7cb70d597caa095e8fa441594b4ba30d"},"cell_type":"code","source":"\"\"\"Sklearn functions that will help training\"\"\"\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e341bb64295768373c3912459b5f76b513acd3d8"},"cell_type":"markdown","source":"**2. Check GPU resources**\nYou can find kernel [documentation here](https://www.kaggle.com/docs/kernels)."},{"metadata":{"trusted":true,"_uuid":"d89aaa8bf16f30a0996a941678118bd483fce08e"},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0df88e72e2573162bb9ef7123b143c8b196f86a9"},"cell_type":"markdown","source":"**3. Globals, file paths, and file inspection**"},{"metadata":{"trusted":true,"_uuid":"e32001c038eecad03422a5d21966915be12ebce8","scrolled":false},"cell_type":"code","source":"VERSION = 'v0'  # Model version\nNUM_CLASSES = 28\nINPUT_SHAPE = [300, 300, 1]\nTRAIN_BATCH_SIZE = 8\nVAL_BATCH_SIZE = 8\n\ndata_path = '/kaggle/input'\ncheckpoint_path = '/kaggle/working'\ntrain_path = os.path.join(data_path, 'train')\ntest_path = os.path.join(data_path,'test')\nlabels_path = os.path.join(data_path, 'train.csv')\nprint(os.listdir(data_path))\ntrain_label_csv = pd.read_csv(labels_path, index_col=False)  # Pandas for reading csv\n\ndef curate_dataset(data_csv):\n    \"\"\"Convert data csv into a list of dicts.\"\"\"\n    dataset = []\n    for name, labels in zip(data_csv.Id, data_csv.Target.str.split(' ')):\n        dataset += [{\n            'path': os.path.join(train_path, name),\n            'labels': np.array([int(label) for label in labels])}]\n    dataset = np.array(dataset)\n    return dataset\n\n\ntrain_dataset = curate_dataset(train_label_csv)\nprint(train_dataset[:10])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b90b37900f061fb8b01af9d9e5e1177d8d886db"},"cell_type":"markdown","source":"**4. Split dataset into training/validation folds for model selection.**"},{"metadata":{"trusted":true,"_uuid":"2c708a1cf712048a95ad98f321e09504d5177780"},"cell_type":"code","source":"train_ids, test_ids, train_targets, test_target = train_test_split(\n    train_label_csv.Id,\n    train_label_csv.Target,\n    test_size=0.1,\n    random_state=42)\n# print(train_ids)\n# print(train_targets)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6d0804b6c6cbb1451a1f57e1e4b9a9c56f66ef3"},"cell_type":"markdown","source":"**5. Create a data generator class for processing and loading data into our model.**\nYou can find a full list of data augmentations [here](https://imgaug.readthedocs.io/en/latest/source/examples_basics.html)."},{"metadata":{"trusted":true,"_uuid":"5f375c5d28052aa4301a0652a566e86ad8e32cfb"},"cell_type":"code","source":"class DataGenerator:\n    \"\"\"Data generator for feeding data to keras\"\"\"\n    def __init__(self,\n            label_dims=28,\n            max_image=255.,\n            batch_size=16,\n            proc_img_size=[300, 300, 4],\n            train=False,\n            imagenet_proc=False):\n        self.label_dims = label_dims\n        self.max_image = max_image\n        self.batch_size = batch_size\n        self.proc_img_size = proc_img_size  # Crop to this size\n        self.train = train\n        self.imagenet_proc = imagenet_proc\n\n    def build(self, dataset_info, augument=True):\n        \"\"\"Data processing routines for training.\"\"\"\n        while True:\n            random_indexes = np.random.choice(len(dataset_info), self.batch_size)\n            batch_images = np.empty(([self.batch_size] + self.proc_img_size))\n            batch_labels = np.zeros((self.batch_size, self.label_dims))\n            for i, idx in enumerate(random_indexes):\n                image = self.load_image(dataset_info[idx]['path']).astype(np.float32)\n                image = self.augmentations(image)\n                image /= self.max_image  # Normalize\n                image = np.maximum(np.minimum(image, 1), 0)  # Clip\n                batch_images[i] = image\n                batch_labels[i][dataset_info[idx]['labels']] = 1\n            yield batch_images, batch_labels\n    \n    def load_image(self, path):\n        \"\"\"Preprocess image.\"\"\"\n        if self.proc_img_size[-1] == 1:\n            image = np.array(Image.open(path + '_green.png'))\n            if len(image.shape) == 2:\n                image = image[..., None]\n        else:\n            R = np.array(Image.open(path + '_red.png'))\n            G = np.array(Image.open(path + '_green.png'))\n            B = np.array(Image.open(path + '_blue.png'))\n            Y = np.array(Image.open(path + '_yellow.png'))\n\n            if self.imagenet_proc:\n                image = np.stack((\n                    R/2 + Y/2, \n                    G/2 + Y/2, \n                    B),-1)\n            else:\n                image = np.stack((R, G, B, Y), axis=-1)\n            # image = cv2.resize(image, (self.proc_img_size[0], self.proc_img_size[1]))\n        return image\n\n    def augmentations(self, image):\n        \"\"\"Apply data augmentations to training images.\"\"\"\n        if self.train:\n            augment_img = iaa.Sequential([\n                iaa.OneOf([\n                    iaa.Affine(rotate=0),\n                    iaa.Affine(rotate=90),\n                    iaa.Affine(rotate=180),\n                    iaa.Affine(rotate=270),\n                    iaa.Fliplr(0.5),\n                    iaa.Flipud(0.5),\n                    # iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)\n                    # iaa.PiecewiseAffine(scale=(0.01, 0.05))\n                ]),\n                iaa.Fliplr(0.5),\n                iaa.Flipud(0.5),\n                iaa.Multiply((0.5, 1.5), per_channel=0.5),\n                iaa.CropToFixedSize(\n                    width=self.proc_img_size[0],\n                    height=self.proc_img_size[1],\n                    position='uniform')],\n            random_order=True)\n        else:\n            augment_img = iaa.Sequential([\n                iaa.CropToFixedSize(\n                    width=self.proc_img_size[0],\n                    height=self.proc_img_size[1],\n                    position='center')])\n        image_aug = augment_img.augment_image(image)\n        return image_aug\n\n    \n# Create train/val datagens\ntrain_datagen = DataGenerator(\n    batch_size=TRAIN_BATCH_SIZE,\n    proc_img_size=INPUT_SHAPE,\n    label_dims=NUM_CLASSES,\n    train=True)\ntrain_datagen = train_datagen.build(\n    dataset_info=train_dataset[train_ids.index])\nval_datagen = DataGenerator(\n    batch_size=VAL_BATCH_SIZE,\n    proc_img_size=INPUT_SHAPE,\n    label_dims=NUM_CLASSES,\n    train=False)\nval_datagen = val_datagen.build(\n    dataset_info=train_dataset[test_ids.index])\n# train_datagen = DataGenerator(proc_img_size=[224, 224, 3], imagenet_proc=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19f794888d66ee7ddcb4c0c34930330e42fbddad"},"cell_type":"markdown","source":"**6. Inspect data.**"},{"metadata":{"trusted":true,"_uuid":"c97f1036c4b55aaddf53d47b3d023b336e307c39"},"cell_type":"code","source":"def plot_images(images, labels, title, num_ims=5):\n    \"\"\"Plot mosaic of images with matplotlib.\"\"\"\n    fig, axs = plt.subplots(1, num_ims, figsize=(25,5))\n    plt.suptitle(title)\n    for idx, (ax, im, lab) in enumerate(zip(axs, images, labels)):\n        ax.imshow(im.squeeze())\n        ax.axis('off')\n        ax.set_title('Label: %s' % np.where(lab)[0])\n    # plt.show()  # Only if not executing in ipython notebook\n    print('{0} range, min: {1}, max: {2}'.format(title, images.min(), images.max()))\n\nimages, labels = next(train_datagen)\nplot_images(images=images, labels=labels, title='Train')\nimages, labels = next(val_datagen)\nplot_images(images=images, labels=labels, title='Val')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d76d298ca9c85386f70db11272d800272d407d38"},"cell_type":"markdown","source":"**7. Build a model!**"},{"metadata":{"trusted":true,"_uuid":"58258d6f30869761152f67ddcaec99fcae69e850","scrolled":false},"cell_type":"code","source":"def resnet_layer(\n        inputs,\n        num_filters=16,\n        kernel_size=3,\n        strides=1,\n        activation='relu',\n        batch_normalization=True,\n        conv_first=True):\n    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n\n    # Arguments\n        inputs (tensor): input tensor from input image or previous layer\n        num_filters (int): Conv2D number of filters\n        kernel_size (int): Conv2D square kernel dimensions\n        strides (int): Conv2D square stride dimensions\n        activation (string): activation name\n        batch_normalization (bool): whether to include batch normalization\n        conv_first (bool): conv-bn-activation (True) or\n            bn-activation-conv (False)\n\n    # Returns\n        x (tensor): tensor as input to the next layer\n    \"\"\"\n    conv = Conv2D(num_filters,\n                  kernel_size=kernel_size,\n                  strides=strides,\n                  padding='same',\n                  kernel_initializer='he_normal',\n                  kernel_regularizer=l2(1e-4))\n\n    x = inputs\n    if conv_first:\n        x = conv(x)\n        if batch_normalization:\n            x = BatchNormalization()(x)\n        if activation is not None:\n            x = Activation(activation)(x)\n    else:\n        if batch_normalization:\n            x = BatchNormalization()(x)\n        if activation is not None:\n            x = Activation(activation)(x)\n        x = conv(x)\n    return x\n\ndef resnet_v2(input_shape, depth, num_classes=NUM_CLASSES):\n    \"\"\"ResNet Version 2 Model builder [b]\n\n    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n    bottleneck layer\n    First shortcut connection per layer is 1 x 1 Conv2D.\n    Second and onwards shortcut connection is identity.\n    At the beginning of each stage, the feature map size is halved (downsampled)\n    by a convolutional layer with strides=2, while the number of filter maps is\n    doubled. Within each stage, the layers have the same number filters and the\n    same filter map sizes.\n    Features maps sizes:\n    conv1  : 32x32,  16\n    stage 0: 32x32,  64\n    stage 1: 16x16, 128\n    stage 2:  8x8,  256\n\n    # Arguments\n        input_shape (tensor): shape of input image tensor\n        depth (int): number of core convolutional layers\n        num_classes (int): number of classes (CIFAR10 has 10)\n\n    # Returns\n        model (Model): Keras model instance\n    \"\"\"\n    if (depth - 2) % 9 != 0:\n        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n    # Start model definition.\n    num_filters_in = 16\n    num_res_blocks = int((depth - 2) / 9)\n\n    inputs = Input(shape=input_shape)\n    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n    x = resnet_layer(inputs=inputs,\n                     num_filters=num_filters_in,\n                     conv_first=True)\n\n    # Instantiate the stack of residual units\n    for stage in range(3):\n        for res_block in range(num_res_blocks):\n            activation = 'relu'\n            batch_normalization = True\n            strides = 1\n            if stage == 0:\n                num_filters_out = num_filters_in * 4\n                if res_block == 0:  # first layer and first stage\n                    activation = None\n                    batch_normalization = False\n            else:\n                num_filters_out = num_filters_in * 2\n                if res_block == 0:  # first layer but not first stage\n                    strides = 2    # downsample\n\n            # bottleneck residual unit\n            y = resnet_layer(inputs=x,\n                             num_filters=num_filters_in,\n                             kernel_size=1,\n                             strides=strides,\n                             activation=activation,\n                             batch_normalization=batch_normalization,\n                             conv_first=False)\n            y = resnet_layer(inputs=y,\n                             num_filters=num_filters_in,\n                             conv_first=False)\n            y = resnet_layer(inputs=y,\n                             num_filters=num_filters_out,\n                             kernel_size=1,\n                             conv_first=False)\n            if res_block == 0:\n                # linear projection residual shortcut connection to match\n                # changed dims\n                x = resnet_layer(inputs=x,\n                                 num_filters=num_filters_out,\n                                 kernel_size=1,\n                                 strides=strides,\n                                 activation=None,\n                                 batch_normalization=False)\n            x = keras.layers.add([x, y])\n        num_filters_in = num_filters_out\n\n    # Add classifier on top.\n    # v2 has BN-ReLU before Pooling\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = AveragePooling2D(pool_size=8)(x)\n    y = Flatten()(x)\n    outputs = Dense(\n        num_classes,\n        activation='softmax',\n        kernel_initializer='he_normal')(y)\n\n    # Instantiate model.\n    model = Model(inputs=inputs, outputs=outputs)\n    return model\n\n\nkeras.backend.clear_session()\nmodel = resnet_v2(input_shape=INPUT_SHAPE, depth=56)\n# print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70cc7a9327c2977dad3c54cea5d8c7bac07634e0"},"cell_type":"markdown","source":"**7. Scoring and history functions for training, and model preparation.**"},{"metadata":{"trusted":true,"_uuid":"115bef252916d4cbd53fba75cfcfc92ab9c060ea"},"cell_type":"code","source":"def f1(y_true, y_pred):\n    \"\"\"Keras function for F1 score, which is the harmonic mean of precision/recall.\"\"\"\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2 * p * r / (p + r + K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef show_history(history):\n    \"\"\"Plot training and validation performance.\"\"\"\n    fig, ax = plt.subplots(1, 3, figsize=(15,5))\n    ax[0].set_title('loss')\n    ax[0].plot(history.epoch, history.history[\"loss\"], label=\"Train loss\")\n    ax[0].plot(history.epoch, history.history[\"val_loss\"], label=\"Validation loss\")\n    ax[1].set_title('f1')\n    ax[1].plot(history.epoch, history.history[\"f1\"], label=\"Train f1\")\n    ax[1].plot(history.epoch, history.history[\"val_f1\"], label=\"Validation f1\")\n    ax[2].set_title('acc')\n    ax[2].plot(history.epoch, history.history[\"acc\"], label=\"Train acc\")\n    ax[2].plot(history.epoch, history.history[\"val_acc\"], label=\"Validation acc\")\n    ax[0].legend()\n    ax[1].legend()\n    ax[2].legend()\n\ncheckpointer = ModelCheckpoint(\n    os.path.join(checkpoint_path, '%s_model.model' % VERSION),\n    verbose=2,\n    save_best_only=True)\nearlyStopping = EarlyStopping(\n    monitor='val_loss',\n    min_delta=0,\n    mode='min',\n    patience=6,\n    verbose=0,\n    restore_best_weights=True)\nreduce_lr = ReduceLROnPlateau(\n    monitor='val_loss',\n    mode='min',\n    factor=0.2,\n    patience=3,\n    min_lr=1e-6,\n    cooldown=1,\n    verbose=1)\nmodel.compile(\n    loss='binary_crossentropy',  \n    optimizer=Adam(1e-3),\n    metrics=['acc', f1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e220516e66553456100c00959c794c41a5896136"},"cell_type":"markdown","source":"**8. Train model.**"},{"metadata":{"trusted":true,"_uuid":"1cc4dd9bb42efd34d3db4d4818193a949b61c4d6"},"cell_type":"code","source":"history = model.fit_generator(\n    train_datagen,\n    steps_per_epoch=20,\n    validation_data=next(val_datagen),\n    epochs=3, \n    verbose=1,\n    callbacks=[checkpointer, earlyStopping, reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f78b693f431caec5ca4c7164e89c2973cb410aa0"},"cell_type":"markdown","source":"**9. Show history and load best model.**"},{"metadata":{"trusted":true,"_uuid":"55be2fd2befa21a1a87227f8164d207b3173dcd6"},"cell_type":"code","source":"show_history(history)\nmodel = load_model(\n    os.path.join(checkpoint_path, '%s_model.model' % VERSION), \n    custom_objects={'f1': f1})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42aa928282600ce7e9432467203a72139ab7ba95"},"cell_type":"markdown","source":"**10. Create submission**"},{"metadata":{"trusted":true,"_uuid":"2d57f9161dec1f78e3b0833a831788091468c801","_kg_hide-output":false},"cell_type":"code","source":"submit = pd.read_csv(os.path.join(data_path, 'sample_submission.csv'))\ntest_dataset = []\nfor name in submit.Id:\n    test_dataset += [{'path': os.path.join(test_path, name), 'labels': 0}]\ntest_dataset = np.array(test_dataset)\n\ntest_datagen = DataGenerator(\n    batch_size=1,\n    proc_img_size=INPUT_SHAPE,\n    label_dims=NUM_CLASSES,\n    train=False)\ntest_datagen = test_datagen.build(\n    dataset_info=test_dataset)\n\npredicted = []\nthreshold = 0.5  # Sigmoid probability\nfor _ in tqdm(test_dataset, total=len(test_dataset), desc='Generating test predictions.'):\n    image = next(test_datagen)[0]\n    score_predict = model.predict(image)[0]\n    label_predict = np.arange(NUM_CLASSES)[score_predict >= threshold]\n    str_predict_label = ' '.join(str(l) for l in label_predict)\n    predicted += [str_predict_label]\nsubmit['Predicted'] = predicted\nsubmit.to_csv('submission.csv', index=False)\nprint(predicted)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}