{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Data Overview\n\nCheck the competition page: https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6/","metadata":{}},{"cell_type":"markdown","source":"## Dependencies","metadata":{}},{"cell_type":"code","source":"# Downgrade tf to prevent errors in mrcnn\n!pip install tensorflow==1.14","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Downgrade keras to prevent errors in mrcnn\n!pip install keras==2.2.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nimport sys\nimport random\nimport math\nimport re\nimport time\nimport numpy as np\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nimport gc\nimport json\nimport glob\nfrom pathlib import Path\nimport keras # to prevent error when importing mrcnn.model\nimport tensorflow as tf\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\n\ntf.__version__\nkeras.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Root and data directory of the project\nROOT_DIR = Path('/kaggle/working')\n\n# Download Mask RCNN library\n!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Mask RCNN\nsys.path.append(ROOT_DIR/'Mask_RCNN')  # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nfrom mrcnn import visualize\nfrom mrcnn.model import log\nimport mrcnn.model as modellib","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Local path to trained weights file\nCOCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n# Download COCO trained weights from Releases if needed\nutils.download_trained_weights(COCO_MODEL_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configurations","metadata":{}},{"cell_type":"code","source":"# Define config class\nclass MyConfig(Config):\n    NAME = \"fashion\"\n    NUM_CLASSES = 46 + 1 # background + 46 classes\n    \n    EPOCHS = 1\n    TRAIN_SIZE = 1/1000\n    VAL_SIZE = 1/10000\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = 128\n    IMAGE_MAX_DIM = 128    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    STEPS_PER_EPOCH = 5\n    VALIDATION_STEPS = 1\n    \n    LEARNING_RATE = 0.002\n\n# Create instance for late use in the model\nconfig = MyConfig()\nconfig.display()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"# Load label json\ntrain_dir = '/kaggle/input/imaterialist-fashion-2019-FGVC6/'\n# Get class names\nwith open(train_dir+'label_descriptions.json') as f:\n    label_descriptions = json.load(f)\nlabel_names = [x['name'] for x in label_descriptions['categories']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load train table\ndf = pd.read_csv(train_dir+'train.csv')\n# Remove attributs from catagory\ndf['labels'] = df['ClassId'].apply(lambda x: x.split('_')[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by image id and concatenate EncodedPixels and labels\ng1_df = df.groupby('ImageId')['EncodedPixels','labels'].agg(lambda x: list(x))\ng2_df = df.groupby('ImageId')['Height', 'Width'].mean()\n\ntrain_df = g1_df.join(g2_df, on='ImageId')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train and validation split\ndf_train,df_val = train_test_split(train_df,train_size=config.TRAIN_SIZE ,test_size=config.VAL_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extend the Dataset class and add load_data() to load the training data. \n# Override the following methods:\n#  load_image()\n#  load_mask()\n#  image_reference()\n\nclass MyDataset(utils.Dataset):\n\n    def load_data(self, label_names, df):\n        # Add classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            \n            self.add_image(\"fashion\", \n                           image_id=i, \n                           path=train_dir+'train/'+i, \n                           labels=row['labels'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], \n                           width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):       \n        img = cv2.imread(self.image_info[image_id]['path'])\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM), interpolation=cv2.INTER_AREA)  \n        return img\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dataset for later use in the model\ntrain_dataset = MyDataset()\ntrain_dataset.load_data(label_names,df_train)\ntrain_dataset.prepare()\n\nvalid_dataset = MyDataset()\nvalid_dataset.load_data(label_names,df_val)\nvalid_dataset.prepare()\n\n# Visualize samples\nfor i in range(2):\n    image_id = random.choice(train_dataset.image_ids)\n    print(train_dataset.image_reference(image_id))\n    \n    image = train_dataset.load_image(image_id)\n    mask, class_ids = train_dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"# Create model in training mode\nmodel = modellib.MaskRCNN(mode=\"training\", config=config, model_dir=ROOT_DIR)\n\n# Load weights trained on MS COCO, but skip layers that are different due to the different number of classes\nmodel.load_weights(COCO_MODEL_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the heads\nmodel.train(train_dataset, valid_dataset, \n            learning_rate=config.LEARNING_RATE, \n            epochs=config.EPOCHS, \n            layers='heads')\nhistory = model.keras_model.history.history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine tune all layers\nmodel.train(train_dataset, valid_dataset, \n            learning_rate=config.LEARNING_RATE, \n            epochs=config.EPOCHS, \n            layers='all')\nnew_history = model.keras_model.history.history\nfor k in new_history: \n    history[k] = history[k] + new_history[k]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detect Objects","metadata":{}},{"cell_type":"code","source":"# Create a configuration for inference\nclass InferenceConfig(MyConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode=\"inference\", \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\n# Get path to saved weights\n# Either set a specific path or find last trained weights\n# model_path = os.path.join(ROOT_DIR, \".h5 file name here\")\nmodel_path = model.find_last()\n\n# Load trained weights\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test on a random image\nimage_id = random.choice(valid_dataset.image_ids)\n\n# Get original image data\noriginal_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n    modellib.load_image_gt(valid_dataset, inference_config, \n                           image_id, use_mini_mask=False)\n\nlog(\"original_image\", original_image)\nlog(\"image_meta\", image_meta)\nlog(\"gt_class_id\", gt_class_id)\nlog(\"gt_bbox\", gt_bbox)\nlog(\"gt_mask\", gt_mask)\n\n# Visualize original image\nvisualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                            train_dataset.class_names, figsize=(8, 8))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict and visualize on the same image\nresults = model.detect([original_image], verbose=1)\n\nr = results[0]\nvisualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                            valid_dataset.class_names, r['scores'],figsize=(8, 8))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"code","source":"\n# Compute VOC-Style mAP @ IoU=0.5\n# Running on 10 images. Increase for better accuracy.\nimage_ids = np.random.choice(valid_dataset.image_ids, 10)\nAPs = []\nfor image_id in image_ids:\n    # Load image and ground truth data\n    image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(valid_dataset, inference_config,\n                               image_id, use_mini_mask=False)\n    molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n    # Run object detection\n    results = model.detect([image], verbose=0)\n    r = results[0]\n    # Compute AP\n    AP, precisions, recalls, overlaps =\\\n        utils.compute_ap(gt_bbox, gt_class_id, gt_mask,\n                         r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'])\n    APs.append(AP)\n    \nprint(\"mAP: \", np.mean(APs))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}