{"cells":[{"metadata":{},"cell_type":"markdown","source":"Welcome to the world where fashion meets computer vision! This is a starter kernel that applies Mask R-CNN with COCO pretrained weights to the task of [iMaterialist (Fashion) 2019 at FGVC6](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport json\nimport glob\nimport random\nfrom pathlib import Path\n\nimport pickle\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\nfrom sklearn.model_selection import StratifiedKFold, KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/imaterialist-fashion-2019-FGVC6')\nROOT_DIR = Path('/kaggle/working')\n\n# For demonstration purpose, the classification ignores attributes (only categories),\n# and the image size is set to 512, which is the same as the size of submission masks\nNUM_CATS = 46\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dowload Libraries and Pretrained Weights","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"sys.path.append(ROOT_DIR/'Mask_RCNN')\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n#!ls -lh mask_rcnn_coco.h5\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp '../input/imat2019-weights/mask_rcnn_fashion_0002.h5'  '/kaggle/working'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COCO_WEIGHTS_PATH = './mask_rcnn_fashion_0002.h5'\nCOCO_WEIGHTS_PATH","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set Config","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Mask R-CNN has a load of hyperparameters. I only adjust some of them.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionConfig(Config):\n    NAME = \"fashion\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4 # a memory error occurs when IMAGES_PER_GPU is too high\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    #DETECTION_NMS_THRESHOLD = 0.0\n    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = 1000\n    VALIDATION_STEPS = 200\n    \nconfig = FashionConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%cd /kaggle/working/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(DATA_DIR/\"label_descriptions.json\") as f:\n    label_descriptions = json.load(f)\n\nlabel_names = [x['name'] for x in label_descriptions['categories']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#label_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(label_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df = pd.read_csv(DATA_DIR/\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#segment_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multilabel_percent = len(segment_df[segment_df['ClassId'].str.contains('_')])/len(segment_df)*100\nprint(f\"Segments that have attributes: {multilabel_percent:.2f}%\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Segments that contain attributes are only 3.46% of data, and [according to the host](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6/discussion/90643#523135), 80% of images have no attribute. So, in the first step, we can only deal with categories to reduce the complexity of the task.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#segment_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df['AttributeId'] = segment_df['ClassId'].str.split('_').str[0]\n\nprint(\"Total segments: \", len(segment_df))\nsegment_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rows with the same image are grouped together because the subsequent operations perform in an image level.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df = segment_df.groupby('ImageId')['EncodedPixels', 'AttributeId'].agg(lambda x: list(x))\nsize_df = segment_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The crucial part is to create a dataset for this task.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#label_descriptions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", i+1, name)\n            \n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"fashion\", \n                           image_id=row.name, \n                           path=str(DATA_DIR/'train'/row.name), \n                           labels=row['AttributeId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize some random images and their masks.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = FashionDataset(image_df)\ndataset.prepare()\n\nfor i in range(6):\n    image_id = random.choice(dataset.image_ids)\n    print(dataset.image_reference(image_id))\n    \n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, the data are partitioned into train and validation sets.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# This code partially supports k-fold training, \n# you can specify the fold to train and the total number of folds here\nFOLD = 0\nN_FOLDS = 5\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits):\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold()\n\ntrain_dataset = FashionDataset(train_df)\ntrain_dataset.prepare()\n\nvalid_dataset = FashionDataset(valid_df)\nvalid_dataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize class distributions of the train and validation data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_segments = np.concatenate(train_df['AttributeId'].values).astype(int)\nprint(\"Total train images: \", len(train_df))\nprint(\"Total train segments: \", len(train_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(train_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\nvalid_segments = np.concatenate(valid_df['AttributeId'].values).astype(int)\nprint(\"Total train images: \", len(valid_df))\nprint(\"Total validation segments: \", len(valid_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(valid_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#the model is trained on epoches 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note that any hyperparameters here, such as LR, may still not be optimal\nLR = 1e-4\nEPOCHS = [2, 6, 8]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This section creates a Mask R-CNN model and specifies augmentations to be used.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"COCO_WEIGHTS_PATH = '/kaggle/working/mask_rcnn_fashion_0002.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('/kaggle/working'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5) # only horizontal flip here\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, we train only the heads.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR*2, # train heads with higher lr to speedup learning\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, all layers are trained.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#import os\n#!cp './fashion20200822T0408/mask_rcnn_fashion_0002.h5' './' \nprint(os.listdir('./'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=2,\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\n#for k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n!cp './fashion20200822T0408/mask_rcnn_fashion_0006.h5' './' \nprint(os.listdir('./'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!cp '/kaggle/working/fashion*/mask_rcnn_fashion_0006.h5' './'\n#print(os.listdir('./'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Afterwards, we reduce LR and train again.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR/5,\n            epochs=EPOCHS[2],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize training history and choose the best epoch.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The final step is to use our model to predict test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"glob_list = glob.glob(f'/kaggle/working/fashion*/mask_rcnn_fashion_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This cell defines InferenceConfig and loads the best trained model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../input/imat2019-weights/mask_rcnn_fashion_0008.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InferenceConfig(FashionConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\nassert model_path != '', \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('trained_MODEL_maskrcn_8k.pkl', 'wb') as fid:\n    pickle.dump(model, fid)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, load the submission data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv(DATA_DIR/\"sample_submission.csv\")\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is the main prediction steps, along with some helper functions.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert data to run-length encoding\ndef to_rle(bits):\n    rle = []\n    pos = 0\n    for bit, group in itertools.groupby(bits):\n        group_list = list(group)\n        if bit:\n            rle.extend([pos, sum(group_list)])\n        pos += len(group_list)\n    return rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Since the submission system does not permit overlapped masks, we have to fix them\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The submission file is created, when all predictions are ready.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Finally, it's pleasing to visualize the results! Sample images contain both fashion models and predictions from the Mask R-CNN model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(3):\n    image_id = sample_df.sample()['ImageId'].values[0]\n    image_path = str(DATA_DIR/'test'/image_id)\n    \n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+label_names, r['scores'],\n                                title=image_id, figsize=(12, 12))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}