{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Welcome to the world where fashion meets computer vision! This is a starter kernel that applies Mask R-CNN with COCO pretrained weights to the task of [iMaterialist (Fashion) 2019 at FGVC6](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6).","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport json\nimport glob\nimport random\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\nfrom sklearn.model_selection import StratifiedKFold, KFold","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:25.501532Z","iopub.execute_input":"2021-08-14T14:54:25.501799Z","iopub.status.idle":"2021-08-14T14:54:27.13741Z","shell.execute_reply.started":"2021-08-14T14:54:25.501748Z","shell.execute_reply":"2021-08-14T14:54:27.136428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/imaterialist-fashion-2019-FGVC6')\nROOT_DIR = Path('/kaggle/working')\n\n# For demonstration purpose, the classification ignores attributes (only categories),\n# and the image size is set to 512, which is the same as the size of submission masks\nNUM_CATS = 46\nIMAGE_SIZE = 512","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:27.13885Z","iopub.execute_input":"2021-08-14T14:54:27.139287Z","iopub.status.idle":"2021-08-14T14:54:27.144166Z","shell.execute_reply.started":"2021-08-14T14:54:27.139238Z","shell.execute_reply":"2021-08-14T14:54:27.143394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dowload Libraries and Pretrained Weights","metadata":{}},{"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-08-14T14:54:27.145802Z","iopub.execute_input":"2021-08-14T14:54:27.146359Z","iopub.status.idle":"2021-08-14T14:54:37.59756Z","shell.execute_reply.started":"2021-08-14T14:54:27.146289Z","shell.execute_reply":"2021-08-14T14:54:37.596647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(ROOT_DIR/'Mask_RCNN')\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-08-14T14:54:37.601388Z","iopub.execute_input":"2021-08-14T14:54:37.60162Z","iopub.status.idle":"2021-08-14T14:54:38.522959Z","shell.execute_reply.started":"2021-08-14T14:54:37.601574Z","shell.execute_reply":"2021-08-14T14:54:38.521161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n#!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = '/kaggle/input/epoch12/12.h5'","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:38.524221Z","iopub.execute_input":"2021-08-14T14:54:38.524522Z","iopub.status.idle":"2021-08-14T14:54:38.528529Z","shell.execute_reply.started":"2021-08-14T14:54:38.524471Z","shell.execute_reply":"2021-08-14T14:54:38.527597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Config","metadata":{}},{"cell_type":"markdown","source":"Mask R-CNN has a load of hyperparameters. I only adjust some of them.","metadata":{}},{"cell_type":"code","source":"class FashionConfig(Config):\n    \"\"\"Configuration for training pneumonia detection on the RSNA pneumonia dataset.\n    Overrides values in the base Config class.\n    \"\"\"\n    \n    # Give the configuration a recognizable name  \n    NAME = 'fashion'\n    \n    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4 \n    \n    BACKBONE = 'resnet50'\n    \n    NUM_CLASSES = 46 + 1  # background + 1 pneumonia classes\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE\n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    TRAIN_ROIS_PER_IMAGE = 32\n    MAX_GT_INSTANCES = 7\n    DETECTION_MAX_INSTANCES = 7\n    DETECTION_MIN_CONFIDENCE = 0.9\n    DETECTION_NMS_THRESHOLD = 0.1\n\n    STEPS_PER_EPOCH = 10000\n    VALIDATION_STEPS = 2000\n    \nconfig = FashionConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:38.529771Z","iopub.execute_input":"2021-08-14T14:54:38.530245Z","iopub.status.idle":"2021-08-14T14:54:38.543429Z","shell.execute_reply.started":"2021-08-14T14:54:38.530195Z","shell.execute_reply":"2021-08-14T14:54:38.542378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Datasets","metadata":{}},{"cell_type":"code","source":"with open(DATA_DIR/\"label_descriptions.json\") as f:\n    label_descriptions = json.load(f)\n\nlabel_names = [x['name'] for x in label_descriptions['categories']]","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:38.54492Z","iopub.execute_input":"2021-08-14T14:54:38.545433Z","iopub.status.idle":"2021-08-14T14:54:38.556685Z","shell.execute_reply.started":"2021-08-14T14:54:38.545214Z","shell.execute_reply":"2021-08-14T14:54:38.555837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segment_df = pd.read_csv(DATA_DIR/\"train.csv\")\nsegment_df = segment_df[segment_df.ClassId != '31'] #Sleeve Dahil Edilmedi\nmultilabel_percent = len(segment_df[segment_df['ClassId'].str.contains('_')])/len(segment_df)*100\nprint(f\"Segments that have attributes: {multilabel_percent:.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:54:38.558187Z","iopub.execute_input":"2021-08-14T14:54:38.558699Z","iopub.status.idle":"2021-08-14T14:55:06.374972Z","shell.execute_reply.started":"2021-08-14T14:54:38.55842Z","shell.execute_reply":"2021-08-14T14:55:06.374041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Segments that contain attributes are only 3.46% of data, and [according to the host](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6/discussion/90643#523135), 80% of images have no attribute. So, in the first step, we can only deal with categories to reduce the complexity of the task.","metadata":{}},{"cell_type":"code","source":"segment_df['CategoryId'] = segment_df['ClassId'].str.split('_').str[0]\n\nprint(\"Total segments: \", len(segment_df))\nsegment_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:06.376333Z","iopub.execute_input":"2021-08-14T14:55:06.376938Z","iopub.status.idle":"2021-08-14T14:55:07.176015Z","shell.execute_reply.started":"2021-08-14T14:55:06.376648Z","shell.execute_reply":"2021-08-14T14:55:07.174996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rows with the same image are grouped together because the subsequent operations perform in an image level.","metadata":{}},{"cell_type":"code","source":"image_df = segment_df.groupby('ImageId')['EncodedPixels', 'CategoryId'].agg(lambda x: list(x))\nsize_df = segment_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:07.177336Z","iopub.execute_input":"2021-08-14T14:55:07.17775Z","iopub.status.idle":"2021-08-14T14:55:16.174481Z","shell.execute_reply.started":"2021-08-14T14:55:07.177577Z","shell.execute_reply":"2021-08-14T14:55:16.17388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is the custom function that resizes an image.","metadata":{}},{"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:16.175636Z","iopub.execute_input":"2021-08-14T14:55:16.175928Z","iopub.status.idle":"2021-08-14T14:55:16.180858Z","shell.execute_reply.started":"2021-08-14T14:55:16.175879Z","shell.execute_reply":"2021-08-14T14:55:16.180031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The crucial part is to create a dataset for this task.","metadata":{}},{"cell_type":"code","source":"class FashionDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"fashion\", \n                           image_id=row.name, \n                           path=str(DATA_DIR/'train'/row.name), \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:16.182182Z","iopub.execute_input":"2021-08-14T14:55:16.182999Z","iopub.status.idle":"2021-08-14T14:55:16.197619Z","shell.execute_reply.started":"2021-08-14T14:55:16.182571Z","shell.execute_reply":"2021-08-14T14:55:16.196642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize some random images and their masks.","metadata":{}},{"cell_type":"code","source":"dataset = FashionDataset(image_df)\ndataset.prepare()\n\nfor i in range(2):\n    image_id = random.choice(dataset.image_ids)\n    print(dataset.image_reference(image_id))\n    \n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:16.19904Z","iopub.execute_input":"2021-08-14T14:55:16.199655Z","iopub.status.idle":"2021-08-14T14:55:23.215645Z","shell.execute_reply.started":"2021-08-14T14:55:16.199353Z","shell.execute_reply":"2021-08-14T14:55:23.214991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, the data are partitioned into train and validation sets.","metadata":{}},{"cell_type":"code","source":"# This code partially supports k-fold training, \n# you can specify the fold to train and the total number of folds here\nFOLD = 0\nN_FOLDS = 10\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits):\n        print(i)\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold()\n\ntrain_dataset = FashionDataset(train_df)\ntrain_dataset.prepare()\n\nvalid_dataset = FashionDataset(valid_df)\nvalid_dataset.prepare()\n\n\n\"------------------------------------------------------TEST------------------------------------------------------------\"\n\n\n\nkf = KFold(n_splits=50, random_state=42, shuffle=True)\nsplits = kf.split(train_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, test_index) in enumerate(splits):\n        print(i)\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[test_index]\n        \n\ntrain_df, test_df = get_fold()\n\ntrain_dataset = FashionDataset(train_df)\ntrain_dataset.prepare()\n\ntest_dataset = FashionDataset(test_df)\ntest_dataset.prepare()\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:23.216802Z","iopub.execute_input":"2021-08-14T14:55:23.217062Z","iopub.status.idle":"2021-08-14T14:55:34.363614Z","shell.execute_reply.started":"2021-08-14T14:55:23.217014Z","shell.execute_reply":"2021-08-14T14:55:34.362933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize class distributions of the train and validation data.","metadata":{}},{"cell_type":"code","source":"train_segments = np.concatenate(train_df['CategoryId'].values).astype(int)\nprint(\"Total train images: \", len(train_df))\nprint(\"Total train segments: \", len(train_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(train_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\nvalid_segments = np.concatenate(valid_df['CategoryId'].values).astype(int)\nprint(\"Total train images: \", len(valid_df))\nprint(\"Total validation segments: \", len(valid_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(valid_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\n\ntest_segments = np.concatenate(test_df['CategoryId'].values).astype(int)\nprint(\"Total train images: \", len(test_df))\nprint(\"Total validation segments: \", len(test_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(test_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:34.364839Z","iopub.execute_input":"2021-08-14T14:55:34.36509Z","iopub.status.idle":"2021-08-14T14:55:36.106871Z","shell.execute_reply.started":"2021-08-14T14:55:34.365049Z","shell.execute_reply":"2021-08-14T14:55:36.106034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Note that any hyperparameters here, such as LR, may still not be optimal\nLR = 1e-4\nEPOCHS = [1]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:36.109742Z","iopub.execute_input":"2021-08-14T14:55:36.109962Z","iopub.status.idle":"2021-08-14T14:55:36.115129Z","shell.execute_reply.started":"2021-08-14T14:55:36.109921Z","shell.execute_reply":"2021-08-14T14:55:36.112971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This section creates a Mask R-CNN model and specifies augmentations to be used.","metadata":{}},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:36.116381Z","iopub.execute_input":"2021-08-14T14:55:36.1168Z","iopub.status.idle":"2021-08-14T14:55:47.392478Z","shell.execute_reply.started":"2021-08-14T14:55:36.116617Z","shell.execute_reply":"2021-08-14T14:55:47.391723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image augmentation\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## rotate\n        iaa.Affine(rotate=0),\n        iaa.Affine(rotate=90),\n        iaa.Affine(rotate=180),\n        iaa.Affine(rotate=270),\n    ]),\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.3)),\n        iaa.Sharpen(alpha=(0.0, 0.3)),\n    ]),\n])","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:47.393897Z","iopub.execute_input":"2021-08-14T14:55:47.394168Z","iopub.status.idle":"2021-08-14T14:55:47.408995Z","shell.execute_reply.started":"2021-08-14T14:55:47.39412Z","shell.execute_reply":"2021-08-14T14:55:47.407882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we train only the heads.","metadata":{}},{"cell_type":"code","source":"#%%time\n#model.train(train_dataset, valid_dataset,\n#            learning_rate=LR*2, # train heads with higher lr to speedup learning\n#            epochs=EPOCHS[0],\n#            layers='heads',\n#            augmentation=None)\n#\n#history = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:47.410363Z","iopub.execute_input":"2021-08-14T14:55:47.410878Z","iopub.status.idle":"2021-08-14T14:55:47.419334Z","shell.execute_reply.started":"2021-08-14T14:55:47.410828Z","shell.execute_reply":"2021-08-14T14:55:47.418658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, all layers are trained.","metadata":{}},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR/5,\n            epochs=EPOCHS[0],\n            layers='all',\n            augmentation=augmentation)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T14:55:47.420975Z","iopub.execute_input":"2021-08-14T14:55:47.421452Z","iopub.status.idle":"2021-08-14T15:03:16.799839Z","shell.execute_reply.started":"2021-08-14T14:55:47.421217Z","shell.execute_reply":"2021-08-14T15:03:16.798391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.keras_model.history.history\n\n\nprint(\"Train Loss: \", history['loss'])\nprint(\"Valid Loss:\",history['val_loss'])\nprint(\"train class loss:\",history['mrcnn_class_loss'])\nprint(\"valid class loss:\",history['val_mrcnn_class_loss'])\nprint(\"train_mask_loss:\",history['mrcnn_mask_loss'])\nprint(\"valid mask loss\",history['val_mrcnn_mask_loss'])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T15:04:29.975192Z","iopub.execute_input":"2021-08-14T15:04:29.975555Z","iopub.status.idle":"2021-08-14T15:04:29.991465Z","shell.execute_reply.started":"2021-08-14T15:04:29.975502Z","shell.execute_reply":"2021-08-14T15:04:29.990088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Afterwards, we reduce LR and train again.","metadata":{}},{"cell_type":"code","source":"#%%time\n#model.train(train_dataset, valid_dataset,\n#            learning_rate=LR/5,\n#            epochs=EPOCHS[2],\n#            layers='all',\n#            augmentation=augmentation)\n#\n#new_history = model.keras_model.history.history\n#for k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2021-08-14T15:03:16.995218Z","iopub.status.idle":"2021-08-14T15:03:16.996853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize training history and choose the best epoch.","metadata":{}},{"cell_type":"code","source":"print(\"It's Done\")","metadata":{"execution":{"iopub.status.busy":"2021-08-14T15:03:16.998745Z","iopub.status.idle":"2021-08-14T15:03:17.003246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}}]}