{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Dataset description\nThis dataset contains images of people wearing a variety of clothing types in a variety of poses. A large number of images and corresponding fashion/apparel segmentations are there in dataset. Images are named with a unique ImageId.\n\n#### Files\n* train/ - training images\n* test/ - test images \n* train.csv - Training annotations, contains images with both segmented apparel categories and fine-grained attributes and images with segmented apparel categories only.\n* label_descriptions.json - gives the apparel categories and fine-grained attributes descriptions.\n\n#### Columns\n* ImageId - unique Id of an image\n* EncodedPixels - run-length encoded format masks.\n* ClassId - class id for mask and represents the apparel category.\n* AttributesIds - attributes ids for mask.","metadata":{}},{"cell_type":"markdown","source":"#### Import libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport json\nimport os\nimport random\nimport cv2\n\nprint(f\"pandas version: {pd.__version__}\")\nprint(f\"numpy version: {np.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:19:39.474279Z","iopub.execute_input":"2022-04-07T13:19:39.474694Z","iopub.status.idle":"2022-04-07T13:19:39.683198Z","shell.execute_reply.started":"2022-04-07T13:19:39.47458Z","shell.execute_reply":"2022-04-07T13:19:39.682395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explorary data analysis","metadata":{}},{"cell_type":"markdown","source":"### Upload datasets","metadata":{}},{"cell_type":"code","source":"# Training dataset\ntrain_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2020-fgvc7/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:19:39.685002Z","iopub.execute_input":"2022-04-07T13:19:39.685275Z","iopub.status.idle":"2022-04-07T13:20:09.609821Z","shell.execute_reply.started":"2022-04-07T13:19:39.685219Z","shell.execute_reply":"2022-04-07T13:20:09.609079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset details","metadata":{}},{"cell_type":"markdown","source":"#### Datasets size","metadata":{}},{"cell_type":"code","source":"# Get datasets shapes\nprint(f'Training dataset shape: {train_df.shape}')\nprint(f'Unique images training: {train_df[\"ImageId\"].nunique()}')","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:09.611437Z","iopub.execute_input":"2022-04-07T13:20:09.611993Z","iopub.status.idle":"2022-04-07T13:20:09.667925Z","shell.execute_reply.started":"2022-04-07T13:20:09.611953Z","shell.execute_reply":"2022-04-07T13:20:09.667082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Size distribution","metadata":{}},{"cell_type":"code","source":"# Get image size distribution\nshape_df = train_df.groupby(\"ImageId\")[[\"Height\", \"Width\"]].first()\nfor dim in [\"Height\", \"Width\"]:\n    plt.figure()\n    plt.hist(shape_df[dim], bins=50)\n    plt.grid()\n    plt.title(f\"{dim} distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:09.670121Z","iopub.execute_input":"2022-04-07T13:20:09.67045Z","iopub.status.idle":"2022-04-07T13:20:10.389948Z","shell.execute_reply.started":"2022-04-07T13:20:09.670411Z","shell.execute_reply":"2022-04-07T13:20:10.389181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Classes per image distribution","metadata":{}},{"cell_type":"code","source":"# Get classes per image distribution\nplt.hist(train_df.groupby(['ImageId'], as_index=False).count()[['ClassId']], bins=30)\nplt.grid()\nplt.title(\"Clases per image distribution distribution\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-07T13:20:10.391423Z","iopub.execute_input":"2022-04-07T13:20:10.391672Z","iopub.status.idle":"2022-04-07T13:20:10.827763Z","shell.execute_reply.started":"2022-04-07T13:20:10.391636Z","shell.execute_reply":"2022-04-07T13:20:10.827073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Plotting random images","metadata":{}},{"cell_type":"code","source":"# Plot randomly selected image\nplt.figure(figsize=(70,7))\nrandom_image = train_df.sample()[\"ImageId\"].item()\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{random_image}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:10.829179Z","iopub.execute_input":"2022-04-07T13:20:10.829435Z","iopub.status.idle":"2022-04-07T13:20:11.352622Z","shell.execute_reply.started":"2022-04-07T13:20:10.829401Z","shell.execute_reply":"2022-04-07T13:20:11.349081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Label description analysis","metadata":{}},{"cell_type":"code","source":"# Get label file\nwith open('/kaggle/input/imaterialist-fashion-2020-fgvc7/label_descriptions.json', 'r') as file:\n    label_d = json.load(file)\n\nprint(\"Label description columns {}\".format(list(label_d.keys())))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.353556Z","iopub.execute_input":"2022-04-07T13:20:11.353805Z","iopub.status.idle":"2022-04-07T13:20:11.367159Z","shell.execute_reply.started":"2022-04-07T13:20:11.353774Z","shell.execute_reply":"2022-04-07T13:20:11.366491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separate label description into categories and attributes\ncategories_df = pd.DataFrame(label_d['categories'])\nattributes_df = pd.DataFrame(label_d['attributes'])","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.368031Z","iopub.execute_input":"2022-04-07T13:20:11.368274Z","iopub.status.idle":"2022-04-07T13:20:11.376124Z","shell.execute_reply.started":"2022-04-07T13:20:11.368217Z","shell.execute_reply":"2022-04-07T13:20:11.375287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Categories\ncategories_df","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.37768Z","iopub.execute_input":"2022-04-07T13:20:11.378603Z","iopub.status.idle":"2022-04-07T13:20:11.397678Z","shell.execute_reply.started":"2022-04-07T13:20:11.378547Z","shell.execute_reply":"2022-04-07T13:20:11.397008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categ_names = categories_df[\"name\"].unique()\nprint(categ_names)\nprint(f\"Number of attributes {len(categ_names)}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.400679Z","iopub.execute_input":"2022-04-07T13:20:11.401143Z","iopub.status.idle":"2022-04-07T13:20:11.406787Z","shell.execute_reply.started":"2022-04-07T13:20:11.401108Z","shell.execute_reply":"2022-04-07T13:20:11.406057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Attributes\npd.set_option('display.max_rows', 500)\nattributes_df","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.408257Z","iopub.execute_input":"2022-04-07T13:20:11.40884Z","iopub.status.idle":"2022-04-07T13:20:11.453783Z","shell.execute_reply.started":"2022-04-07T13:20:11.408707Z","shell.execute_reply":"2022-04-07T13:20:11.453049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attr_names = attributes_df[\"name\"].unique()\nprint(attr_names)\nprint(f\"Number of attributes {len(attr_names)}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.455077Z","iopub.execute_input":"2022-04-07T13:20:11.455425Z","iopub.status.idle":"2022-04-07T13:20:11.461992Z","shell.execute_reply.started":"2022-04-07T13:20:11.455392Z","shell.execute_reply":"2022-04-07T13:20:11.461214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dictionaries to map the IDs with the category and attributes strings\ncat_map = {category[\"id\"]: category[\"name\"] for category in label_d['categories']}\ncat_map_inv = {category[\"name\"]: category[\"id\"] for category in label_d['categories']}\n\nattr_map = {category[\"id\"]: category[\"name\"] for category in label_d['attributes']}\nattr_map_inv = {category[\"name\"]: category[\"id\"] for category in label_d['attributes']}","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.463604Z","iopub.execute_input":"2022-04-07T13:20:11.464277Z","iopub.status.idle":"2022-04-07T13:20:11.470721Z","shell.execute_reply.started":"2022-04-07T13:20:11.464118Z","shell.execute_reply":"2022-04-07T13:20:11.469961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Plot segmented images","metadata":{}},{"cell_type":"code","source":"def plot_raw_segmented_image(df, figsize=(15,15)):\n    # Read random image\n    random_id = df.sample()[\"ImageId\"].item()\n    image = mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{random_id}.jpg')\n    shape = image.shape,\n    encoded_pixels = df[train_df['ImageId'] == random_id]['EncodedPixels']\n    class_ids = df[train_df['ImageId'] == random_id]['ClassId']\n    \n    # Create mask\n    height, width = shape[0][:2]\n    mask = np.zeros((height, width)).reshape(-1)\n    for pixels, class_id in zip(encoded_pixels, class_ids):\n        pixels_split = list(map(int, pixels.split()))\n        pixel_starts = pixels_split[::2]\n        run_lengths = pixels_split[1::2]\n        for pixel_start, run_length in zip(pixel_starts, run_lengths):\n            mask[pixel_start:pixel_start + run_length] = 255 - class_id * 4\n    mask = mask.reshape(height, width, order='F')    \n    \n    # Plot images\n    fig, axs = plt.subplots(1, 2,figsize=(15,15))\n    axs[0].imshow(image)    \n    axs[1].imshow(image)    \n    axs[1].imshow(mask, alpha=0.8)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.472436Z","iopub.execute_input":"2022-04-07T13:20:11.472969Z","iopub.status.idle":"2022-04-07T13:20:11.487036Z","shell.execute_reply.started":"2022-04-07T13:20:11.472932Z","shell.execute_reply":"2022-04-07T13:20:11.486135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot raw and segmented images\nsize = 3\nfor _ in range(size):\n    plot_raw_segmented_image(train_df, \"ImageId\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:11.488871Z","iopub.execute_input":"2022-04-07T13:20:11.489636Z","iopub.status.idle":"2022-04-07T13:20:23.45364Z","shell.execute_reply.started":"2022-04-07T13:20:11.489593Z","shell.execute_reply":"2022-04-07T13:20:23.451292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_classes_image(df, figsize=(15,15)):\n    # Select random image\n    random_id = df.sample()[\"ImageId\"].item()\n    image = mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{random_id}.jpg')\n    shape = image.shape,\n    encoded_pixels = df[train_df['ImageId'] == random_id]['EncodedPixels']\n    class_ids = df[train_df['ImageId'] == random_id]['ClassId']\n    \n    # Create mask and plot every specific class in the image\n    height, width = shape[0][:2]\n    for pixels, class_id in zip(encoded_pixels, class_ids):\n        mask = np.zeros((height, width)).reshape(-1)\n        pixels_split = list(map(int, pixels.split()))\n        pixel_starts = pixels_split[::2]\n        run_lengths = pixels_split[1::2]\n        for pixel_start, run_length in zip(pixel_starts, run_lengths):\n            mask[pixel_start:pixel_start + run_length] = 255 - class_id * 4\n        mask = mask.reshape(height, width, order='F')\n        \n        # Plot masked image\n        plt.figure(figsize=(15, 15))\n        plt.title(cat_map[class_id])\n        plt.imshow(image)    \n        plt.imshow(mask, alpha=0.8)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:23.455087Z","iopub.execute_input":"2022-04-07T13:20:23.455574Z","iopub.status.idle":"2022-04-07T13:20:23.466892Z","shell.execute_reply.started":"2022-04-07T13:20:23.45554Z","shell.execute_reply":"2022-04-07T13:20:23.4661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_classes_image(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:23.468531Z","iopub.execute_input":"2022-04-07T13:20:23.469026Z","iopub.status.idle":"2022-04-07T13:20:27.533782Z","shell.execute_reply.started":"2022-04-07T13:20:23.468989Z","shell.execute_reply":"2022-04-07T13:20:27.533104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Replace ClassId value","metadata":{}},{"cell_type":"code","source":"# Replace ClassId for class string \ntrain_df['ClassId'] = train_df['ClassId'].map(cat_map)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:27.535093Z","iopub.execute_input":"2022-04-07T13:20:27.535561Z","iopub.status.idle":"2022-04-07T13:20:27.562492Z","shell.execute_reply.started":"2022-04-07T13:20:27.535527Z","shell.execute_reply":"2022-04-07T13:20:27.561792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot class value count\ntrain_df['ClassId'].value_counts()[:20].plot(kind='barh')\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:27.563954Z","iopub.execute_input":"2022-04-07T13:20:27.564466Z","iopub.status.idle":"2022-04-07T13:20:27.925406Z","shell.execute_reply.started":"2022-04-07T13:20:27.564428Z","shell.execute_reply":"2022-04-07T13:20:27.924736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform ClassId back to int to perform the training\ntrain_df['ClassId'] = train_df['ClassId'].map(cat_map_inv)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:27.926679Z","iopub.execute_input":"2022-04-07T13:20:27.927065Z","iopub.status.idle":"2022-04-07T13:20:27.963773Z","shell.execute_reply.started":"2022-04-07T13:20:27.927029Z","shell.execute_reply":"2022-04-07T13:20:27.962996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detectron","metadata":{}},{"cell_type":"markdown","source":"#### Install dependencies","metadata":{}},{"cell_type":"code","source":"!pip install -q cython pyyaml","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:27.965049Z","iopub.execute_input":"2022-04-07T13:20:27.965662Z","iopub.status.idle":"2022-04-07T13:20:36.527046Z","shell.execute_reply.started":"2022-04-07T13:20:27.965624Z","shell.execute_reply":"2022-04-07T13:20:36.52621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pycocotools==2.0.2","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:36.529738Z","iopub.execute_input":"2022-04-07T13:20:36.530034Z","iopub.status.idle":"2022-04-07T13:20:53.549181Z","shell.execute_reply.started":"2022-04-07T13:20:36.529994Z","shell.execute_reply":"2022-04-07T13:20:53.548355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install 'git+https://github.com/facebookresearch/detectron2.git'","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:20:53.551653Z","iopub.execute_input":"2022-04-07T13:20:53.55211Z","iopub.status.idle":"2022-04-07T13:23:42.76278Z","shell.execute_reply.started":"2022-04-07T13:20:53.552069Z","shell.execute_reply":"2022-04-07T13:23:42.761913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Import libraries","metadata":{}},{"cell_type":"code","source":"from detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom detectron2.structures import BoxMode","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:23:42.764281Z","iopub.execute_input":"2022-04-07T13:23:42.764889Z","iopub.status.idle":"2022-04-07T13:23:43.909326Z","shell.execute_reply.started":"2022-04-07T13:23:42.764847Z","shell.execute_reply":"2022-04-07T13:23:43.908611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Use detectron in current dataset","metadata":{}},{"cell_type":"code","source":"# Use detectron to predict images in the current dataset,\n# use the default weights and labels from the network\nconfig_file = \"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(config_file))\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_file)\n\npredictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:23:43.910791Z","iopub.execute_input":"2022-04-07T13:23:43.911083Z","iopub.status.idle":"2022-04-07T13:23:56.853292Z","shell.execute_reply.started":"2022-04-07T13:23:43.911045Z","shell.execute_reply":"2022-04-07T13:23:56.852528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot images classified by the pretrained detectron\nrows, cols = 2, 2\nplt.figure(figsize=(20, 20))\n\nfor i in range(int(rows * cols)):\n    plt.subplot(rows, cols, i + 1)\n    \n    # Get random image\n    random_id = train_df.sample()[\"ImageId\"].item()\n    im = mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{random_id}.jpg')\n    height, width = im.shape[:2]\n    \n    # Get detectron prediction from the selected image\n    outputs = predictor(im)\n    \n    # Create visualizer\n    visualizer = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=0.4)\n    \n    # Change font size for better reading\n    visualizer._default_font_size = np.sqrt(height * width) // 20\n    visualizer = visualizer.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    \n    # Plot images\n    plt.axis('off')\n    plt.imshow(visualizer.get_image()[:, :, ::-1])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:23:56.854558Z","iopub.execute_input":"2022-04-07T13:23:56.854816Z","iopub.status.idle":"2022-04-07T13:24:04.270431Z","shell.execute_reply.started":"2022-04-07T13:23:56.854781Z","shell.execute_reply":"2022-04-07T13:24:04.269643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Modify dataset to use detectron","metadata":{}},{"cell_type":"code","source":"\ndef rle_decode_string(string, h, w):\n    \"\"\"\n    Transforms rle string into a pixel mask\n    \n    :param string: rle string to transform into mask\n    :type string: str\n    :param string: image height\n    :type string: int\n    :param string: image width\n    :type string: int\n    :return: image mask\n    :rtype: numpy array\n\n    \"\"\"\n    mask = np.full(h * w, 0, dtype=np.uint8)\n    annotation = [int(x) for x in string.split(' ')]\n    for i, start_pixel in enumerate(annotation[::2]):\n        mask[start_pixel: start_pixel + annotation[2 * i + 1]] = 1\n    mask = mask.reshape((h, w), order='F')\n    return mask\n\ndef rle2bbox(rle, shape):\n    '''\n    Get a bbox from a mask which is required for Detectron 2 dataset\n    :param rle: run-length encoded image mask, as string\n    :type rle: str\n    :param shape: (height, width) of image on which RLE was produced\n    :type rle: tuple\n    :return: (x0, y0, x1, y1) tuple describing the bounding box of the rle mask\n    :rtype: tuple\n    '''\n    \n    a = np.fromiter(rle.split(), dtype=np.uint)\n    a = a.reshape((-1, 2))  # an array of (start, length) pairs\n    a[:,0] -= 1  # `start` is 1-indexed\n    \n    y0 = a[:,0] % shape[0]\n    y1 = y0 + a[:,1]\n    if np.any(y1 > shape[0]):\n        # got `y` overrun, meaning that there are a pixels in mask on 0 and shape[0] position\n        y0 = 0\n        y1 = shape[0]\n    else:\n        y0 = np.min(y0)\n        y1 = np.max(y1)\n    \n    x0 = a[:,0] // shape[0]\n    x1 = (a[:,0] + a[:,1]) // shape[0]\n    x0 = np.min(x0)\n    x1 = np.max(x1)\n    \n    if x1 > shape[1]:\n        # just went out of the image dimensions\n        raise ValueError(\"invalid RLE or image dimensions: x1=%d > shape[1]=%d\" % (\n            x1, shape[1]\n        ))\n\n    return x0, y0, x1, y1","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:24:04.271671Z","iopub.execute_input":"2022-04-07T13:24:04.272007Z","iopub.status.idle":"2022-04-07T13:24:04.285931Z","shell.execute_reply.started":"2022-04-07T13:24:04.271976Z","shell.execute_reply":"2022-04-07T13:24:04.285328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform ImageId into image path\nimage_dir = '/kaggle/input/imaterialist-fashion-2020-fgvc7/train/'\ntrain_df['ImageId'] = image_dir + train_df['ImageId'] + '.jpg'\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:24:04.290761Z","iopub.execute_input":"2022-04-07T13:24:04.291133Z","iopub.status.idle":"2022-04-07T13:24:04.439119Z","shell.execute_reply.started":"2022-04-07T13:24:04.291095Z","shell.execute_reply":"2022-04-07T13:24:04.438296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create boxes list\nbboxes = [rle2bbox(c.EncodedPixels, (c.Height, c.Width)) for n, c in train_df.iterrows()]\nbboxes_array = np.array(bboxes)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:24:04.440646Z","iopub.execute_input":"2022-04-07T13:24:04.440906Z","iopub.status.idle":"2022-04-07T13:25:47.077022Z","shell.execute_reply.started":"2022-04-07T13:24:04.440871Z","shell.execute_reply":"2022-04-07T13:25:47.076211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill NaNs\ntrain_df = train_df.fillna(999)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:25:47.078183Z","iopub.execute_input":"2022-04-07T13:25:47.0788Z","iopub.status.idle":"2022-04-07T13:25:47.265165Z","shell.execute_reply.started":"2022-04-07T13:25:47.078757Z","shell.execute_reply":"2022-04-07T13:25:47.264419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add bounding boxes coordinates to train using detectron\ntrain_df['x0'], train_df['y0'], train_df['x1'], train_df['y1'] = bboxes_array[:,0], bboxes_array[:,1], bboxes_array[:,2], bboxes_array[:,3]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:25:47.26663Z","iopub.execute_input":"2022-04-07T13:25:47.26715Z","iopub.status.idle":"2022-04-07T13:25:47.292908Z","shell.execute_reply.started":"2022-04-07T13:25:47.267108Z","shell.execute_reply":"2022-04-07T13:25:47.292295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_to_array(value):\n    if isinstance(value, (np.ndarray, np.generic)):\n        return value\n    elif isinstance(value, str):\n        array = [int(val) for val in value.split(\",\")]\n    elif isinstance(value, int):\n        array = [999] \n    array = np.array(array)\n    return np.pad(array, (0, 14 - len(array)))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:25:47.294307Z","iopub.execute_input":"2022-04-07T13:25:47.294772Z","iopub.status.idle":"2022-04-07T13:25:47.30177Z","shell.execute_reply.started":"2022-04-07T13:25:47.294738Z","shell.execute_reply":"2022-04-07T13:25:47.301157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform attribute string into tensor\ntrain_df[\"AttributesIds\"] = train_df[\"AttributesIds\"].map(transform_to_array)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:25:47.303349Z","iopub.execute_input":"2022-04-07T13:25:47.303905Z","iopub.status.idle":"2022-04-07T13:25:59.716204Z","shell.execute_reply.started":"2022-04-07T13:25:47.303868Z","shell.execute_reply":"2022-04-07T13:25:59.715509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Store modified train_df\ntrain_df.to_pickle(\"train_df.pickle\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:25:59.717633Z","iopub.execute_input":"2022-04-07T13:25:59.717903Z","iopub.status.idle":"2022-04-07T13:26:04.039271Z","shell.execute_reply.started":"2022-04-07T13:25:59.717868Z","shell.execute_reply":"2022-04-07T13:26:04.038474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_df))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:26:04.04075Z","iopub.execute_input":"2022-04-07T13:26:04.040999Z","iopub.status.idle":"2022-04-07T13:26:04.047048Z","shell.execute_reply.started":"2022-04-07T13:26:04.040966Z","shell.execute_reply":"2022-04-07T13:26:04.046294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycocotools\ndef get_materialist_dicts(df):\n    \"\"\"\n    Transforms dataframe into dictionary used to train using detectron\n    \"\"\"\n    dataset_dicts = []\n    for idx, filename in enumerate(df[\"ImageId\"].unique()):\n        record = {}\n        # Get useful image information\n        height, width = df[df[\"ImageId\"] == filename][[\"Height\", \"Width\"]].values[0]\n        record[\"file_name\"] = filename\n        record[\"image_id\"] = idx\n        record[\"height\"] = int(height)\n        record[\"width\"] = int(width)\n        \n        if idx % 1000 == 0:\n            print(idx)\n        \n        objs = []\n        for i, row in df[(df['ImageId'] == filename)].iterrows():\n            \n            # Get segmentation polygons\n            mask = rle_decode_string(row['EncodedPixels'], row['Height'], row['Width'])\n            # segmentation = pycocotools.mask.encode(np.asarray(mask, order=\"F\"))\n            contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE,\n                                                    cv2.CHAIN_APPROX_SIMPLE)\n            segmentation = []\n\n            for contour in contours:\n                contour = contour.flatten().tolist()\n                if len(contour) > 4:\n                    segmentation.append(contour)\n\n            obj = {\n                \"bbox\": [row['x0'], row['y0'], row['x1'], row['y1']],\n                \"bbox_mode\": BoxMode.XYXY_ABS,\n                \"segmentation\": segmentation,\n                \"category_id\": row['ClassId'],\n                \"attributes\": row['AttributesIds'],\n                \"iscrowd\": 0,\n            }\n            objs.append(obj)\n        \n        record['annotations'] = objs\n        dataset_dicts.append(record)\n    return dataset_dicts\n\n# Use reduced dictionary to reduce the time to transform into dictionaries\n#df_copy = train_df[:8000].copy()\n\ndf_copy = train_df.copy()\n\ndf_copy = train_df[:23000].copy()\ndf_copy_val = train_df[23000:24000].copy()\n\n# Full dictionary\n# df_copy = train_df.copy()\n\nmaterialist_dict = get_materialist_dicts(df_copy)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-07T13:26:04.049283Z","iopub.execute_input":"2022-04-07T13:26:04.049531Z","iopub.status.idle":"2022-04-07T13:34:40.235496Z","shell.execute_reply.started":"2022-04-07T13:26:04.049497Z","shell.execute_reply":"2022-04-07T13:34:40.234671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"materialist_dict[0].keys()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:40.236958Z","iopub.execute_input":"2022-04-07T13:34:40.237224Z","iopub.status.idle":"2022-04-07T13:34:40.244127Z","shell.execute_reply.started":"2022-04-07T13:34:40.237189Z","shell.execute_reply":"2022-04-07T13:34:40.243393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(df_copy))\nprint(len(materialist_dict))\nprint(len(train_df))\nprint(len(train_df[\"ImageId\"].unique()))\nprint(len(df_copy[\"ImageId\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:40.24549Z","iopub.execute_input":"2022-04-07T13:34:40.245962Z","iopub.status.idle":"2022-04-07T13:34:40.341102Z","shell.execute_reply.started":"2022-04-07T13:34:40.245919Z","shell.execute_reply":"2022-04-07T13:34:40.340305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Register the custom dataset to detectron2,\nfor d in [\"train\", \"val\"]:\n    if d == \"train\":\n        used_df = df_copy\n    else:\n        used_df = df_copy_val\n    DatasetCatalog.register(\"mat_\" + d, lambda df=used_df: get_materialist_dicts(df))\n    # DatasetCatalog.register(\"mat_\" + d, lambda df=df_copy: get_materialist_dicts(df))\n    MetadataCatalog.get(\"mat_\" + d).set(thing_classes=list(categories_df.name))\nmaterialist_metadata = MetadataCatalog.get(\"mat_train\")\n","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:40.342552Z","iopub.execute_input":"2022-04-07T13:34:40.344413Z","iopub.status.idle":"2022-04-07T13:34:40.351151Z","shell.execute_reply.started":"2022-04-07T13:34:40.344368Z","shell.execute_reply":"2022-04-07T13:34:40.350296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(materialist_metadata)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:40.352857Z","iopub.execute_input":"2022-04-07T13:34:40.353526Z","iopub.status.idle":"2022-04-07T13:34:40.361953Z","shell.execute_reply.started":"2022-04-07T13:34:40.353484Z","shell.execute_reply":"2022-04-07T13:34:40.360968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To verify the data loading is correct we visualize the annotations of randomly selected samples in the training set\nfor d in random.sample(materialist_dict, 5):\n    img = cv2.imread(d[\"file_name\"])\n    img = mpimg.imread(d[\"file_name\"])\n    height, width = img.shape[:2]\n    plt.figure(figsize=(20, 20))\n    visualizer = Visualizer(img[:, :, ::-1], metadata=materialist_metadata, scale=0.5)\n    visualizer._default_font_size = np.sqrt(height * width) // 20\n    out = visualizer.draw_dataset_dict(d)\n    plt.imshow(out.get_image()[:, :, ::-1])\n    plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:40.363847Z","iopub.execute_input":"2022-04-07T13:34:40.364649Z","iopub.status.idle":"2022-04-07T13:34:43.739035Z","shell.execute_reply.started":"2022-04-07T13:34:40.364609Z","shell.execute_reply":"2022-04-07T13:34:43.738433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FPN","metadata":{}},{"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n\n# Fine-tune a COCO-pretrained R50-FPN Mask R-CNN model on the dataset\ncfg_FPN = get_cfg()\ncfg_FPN.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg_FPN.DATASETS.TRAIN = (\"mat_train\",)\ncfg_FPN.DATASETS.TEST = (\"mat_val\",)\ncfg_FPN.DATALOADER.NUM_WORKERS = 1\ncfg_FPN.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\ncfg_FPN.SOLVER.IMS_PER_BATCH = 2\ncfg_FPN.SOLVER.BASE_LR = 0.00025 \ncfg_FPN.SOLVER.MAX_ITER = 1000  \ncfg_FPN.SOLVER.STEPS = []       \ncfg_FPN.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128  \ncfg_FPN.MODEL.ROI_HEADS.NUM_CLASSES = 46 \n\n# Train\ncfg_FPN.OUTPUT_DIR = \"./output_FPN\"\nos.makedirs(cfg_FPN.OUTPUT_DIR, exist_ok=True)\ntrainer_FPN = DefaultTrainer(cfg_FPN) \ntrainer_FPN.resume_or_load(resume=False)\ntrainer_FPN.train()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T13:34:43.740093Z","iopub.execute_input":"2022-04-07T13:34:43.740909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create predictor from the weigths obtained during the training\ncfg_FPN.MODEL.WEIGHTS = os.path.join(cfg_FPN.OUTPUT_DIR, \"model_final.pth\")\ncfg_FPN.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model\ncfg_FPN.DATASETS.TEST = ('mat_val',)\npredictor_FPN = DefaultPredictor(cfg_FPN)\nf = open('configFPN.yml', 'w')\nf.write(cfg.dump())\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\nplt.figure(figsize=(20,20))\nfor d in random.sample(materialist_dict, 3):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor_FPN(im)\n    visualizer = Visualizer(im[:, :, ::-1],\n                   metadata=materialist_metadata, \n                   scale=0.8, \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n    )\n    v = visualizer.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(v.get_image()[:, :, ::-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(\"/kaggle/input/trial-images/img1.jpeg\")\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nouts = predictor_FPN(img)\nvisualizer = Visualizer(img[:, :, ::-1],\n               metadata=materialist_metadata, \n               scale=0.8, \n               instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n)\nv = visualizer.draw_instance_predictions(outs[\"instances\"].to(\"cpu\"))\nplt.imshow(v.get_image()[:, :, ::-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img2 = cv2.imread(\"/kaggle/input/trial-images/img2.jpeg\")\nimg2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)\nouts = predictor_FPN(img2)\nvisualizer = Visualizer(img2[:, :, ::-1],\n               metadata=materialist_metadata, \n               scale=0.8, \n               instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n)\nv = visualizer.draw_instance_predictions(outs[\"instances\"].to(\"cpu\"))\nplt.imshow(v.get_image()[:, :, ::-1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\n\n# Show different images at random\nrows, cols = 3, 3\nplt.figure(figsize=(20,20))\n\nfor i, d in enumerate(random.sample(materialist_dict, 9)):\n    # Process image\n    plt.subplot(rows, cols, i+1)\n\n    im = cv2.imread(d[\"file_name\"])\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    \n    # Run through predictor\n    outputs = predictor_FPN(im)\n    \n    # Visualize\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=materialist_metadata, \n                   scale=0.8, \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n    )\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(v.get_image()[:, :, ::-1])\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.data import build_detection_test_loader\n\n# Evaluate model\nevaluator_FPN = COCOEvaluator(\"mat_val\", output_dir=\"./output\")\nval_loader_FPN = build_detection_test_loader(cfg_FPN, \"mat_val\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get results\nresult_FPN = inference_on_dataset(predictor_FPN.model, val_loader_FPN, evaluator_FPN)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_FPN","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DC5","metadata":{}},{"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n\n# Fine-tune a COCO-pretrained R50-DC5 Mask R-CNN model on the dataset\ncfg_DC5 = get_cfg()\ncfg_DC5.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x.yaml\"))\ncfg_DC5.DATASETS.TRAIN = (\"mat_train\",)\ncfg_DC5.DATASETS.TEST = (\"mat_val\",)\ncfg_DC5.DATALOADER.NUM_WORKERS = 1\ncfg_DC5.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x.yaml\")\ncfg_DC5.SOLVER.IMS_PER_BATCH = 2\ncfg_DC5.SOLVER.BASE_LR = 0.00025 \ncfg_DC5.SOLVER.MAX_ITER = 1000  \ncfg_DC5.SOLVER.STEPS = []       \ncfg_DC5.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128  \ncfg_DC5.MODEL.ROI_HEADS.NUM_CLASSES = 46 \n\n# Train\ncfg_DC5.OUTPUT_DIR = \"./output_DC5\"\nos.makedirs(cfg_DC5.OUTPUT_DIR, exist_ok=True)\ntrainer_DC5 = DefaultTrainer(cfg_DC5) \ntrainer_DC5.resume_or_load(resume=False)\ntrainer_DC5.train()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create predictor from the weigths obtained during the training\ncfg_DC5.MODEL.WEIGHTS = os.path.join(cfg_DC5.OUTPUT_DIR, \"model_final.pth\")\ncfg_DC5.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model\ncfg_DC5.DATASETS.TEST = ('mat_val',)\npredictor_DC5 = DefaultPredictor(cfg_DC5)\nf = open('configDC5.yml', 'w')\nf.write(cfg.dump())\nf.close()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loader_DC5 = build_detection_test_loader(cfg_DC5, \"mat_val\")\n# Get results\nresult_DC5 = inference_on_dataset(predictor_DC5.model, val_loader_DC5, evaluator_FPN)\nresult_DC5","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## C4","metadata":{}},{"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n\n# Fine-tune a COCO-pretrained R50-C4 Mask R-CNN model on the dataset\ncfg_C4 = get_cfg()\ncfg_C4.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml\"))\ncfg_C4.DATASETS.TRAIN = (\"mat_train\",)\ncfg_C4.DATASETS.TEST = (\"mat_val\",)\ncfg_C4.DATALOADER.NUM_WORKERS = 1\ncfg_C4.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml\")\ncfg_C4.SOLVER.IMS_PER_BATCH = 2\ncfg_C4.SOLVER.BASE_LR = 0.00025 \ncfg_C4.SOLVER.MAX_ITER = 1000  \ncfg_C4.SOLVER.STEPS = []       \ncfg_C4.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128  \ncfg_C4.MODEL.ROI_HEADS.NUM_CLASSES = 46 \n\n# Train\ncfg_C4.OUTPUT_DIR = \"./output_C4\"\nos.makedirs(cfg_C4.OUTPUT_DIR, exist_ok=True)\ntrainer_C4 = DefaultTrainer(cfg_C4) \ntrainer_C4.resume_or_load(resume=False)\ntrainer_C4.train()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create predictor from the weigths obtained during the training\ncfg_C4.MODEL.WEIGHTS = os.path.join(cfg_C4.OUTPUT_DIR, \"model_final.pth\")\ncfg_C4.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model\ncfg_C4.DATASETS.TEST = ('mat_val',)\npredictor_C4 = DefaultPredictor(cfg_C4)\nf = open('configC4.yml', 'w')\nf.write(cfg.dump())\nf.close()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loader_C4 = build_detection_test_loader(cfg_C4, \"mat_val\")\n# Get results\nresult_C4 = inference_on_dataset(predictor_C4.model, val_loader_C4, evaluator_FPN)\nresult_C4","metadata":{},"execution_count":null,"outputs":[]}]}