{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Installing Detectron and dependencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U torch==1.5 torchvision==0.6 -f https://download.pytorch.org/whl/cu101/torch_stable.html ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!python -m pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/index.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames[:5]:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import collections\nimport torch\nimport json\nimport os\nimport cv2\nimport random\nimport gc\nimport pycocotools\nimport torch.nn.functional as F\n\nimport numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport PIL\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom pathlib import Path\n\n# import some common detectron2 utilities\nfrom detectron2.structures import BoxMode\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Examples with pretrained Instance Segmentation"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cfg = get_cfg()\n# Merging model configs with default \ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n\n# Get weights from Instance segmentation Mask RCNN R 50 FPN model\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\n\npredictor = DefaultPredictor(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_list = []\nfor dirname, _, filenames in os.walk('/kaggle/input/imaterialist-fashion-2020-fgvc7/train/'):\n    for filename in filenames:\n        images_list.append(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show different images at random\nrows, cols = 3, 3\nplt.figure(figsize=(20,20))\n\nfor i, image in enumerate(random.sample(images_list, 9)):\n    \n    # Process image\n    im = cv2.imread(image)\n    plt.subplot(rows, cols, i+1)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    \n    # Run through predictor\n    outputs = predictor(im)\n    \n    # Visualize\n    v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.axis('off')\n    plt.imshow(v.get_image()[:, :, ::-1])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Examples with Keypoint Detection "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cfg = get_cfg()\n\ncfg.merge_from_file(model_zoo.get_config_file(\"Misc/cascade_mask_rcnn_R_50_FPN_1x.yaml\"))\n\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n\n# Get weights from Keypoint Mask RCNN R 50 FPN model\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"Misc/cascade_mask_rcnn_R_50_FPN_1x.yaml\")\n\npredictor = DefaultPredictor(cfg)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show different images at random\nrows, cols = 3, 4\nplt.figure(figsize=(20,30))\n\nfor i, image in enumerate(random.sample(images_list, 12)):\n    \n    # Process image\n    im = cv2.imread(image)\n    plt.subplot(cols, rows, i+1)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    \n    # Run through predictor\n    outputs = predictor(im)\n    \n    # Visualize\n    v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.axis('off')\n    plt.imshow(v.get_image()[:, :, ::-1])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = Path('/kaggle/input/imaterialist-fashion-2020-fgvc7/')\nimage_dir = Path('/kaggle/input/imaterialist-fashion-2020-fgvc7/train/')\ndf = pd.read_csv(data_dir/'train.csv')\n\n# Get label descriptions\nwith open(data_dir/'label_descriptions.json', 'r') as file:\n    label_desc = json.load(file)\ndf_categories = pd.DataFrame(label_desc['categories'])\ndf_attributes = pd.DataFrame(label_desc['attributes'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rle helper functions\n\ndef rle_decode_string(string, h, w):\n    mask = np.full(h*w, 0, dtype=np.uint8)\n    annotation = [int(x) for x in string.split(' ')]\n    for i, start_pixel in enumerate(annotation[::2]):\n        mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n    mask = mask.reshape((h, w), order='F')\n\n    \n    return mask\n\ndef rle2bbox(rle, shape):\n    '''\n    Get a bbox from a mask which is required for Detectron 2 dataset\n    rle: run-length encoded image mask, as string\n    shape: (height, width) of image on which RLE was produced\n    Returns (x0, y0, x1, y1) tuple describing the bounding box of the rle mask\n    \n    Note on image vs np.array dimensions:\n    \n        np.array implies the `[y, x]` indexing order in terms of image dimensions,\n        so the variable on `shape[0]` is `y`, and the variable on the `shape[1]` is `x`,\n        hence the result would be correct (x0,y0,x1,y1) in terms of image dimensions\n        for RLE-encoded indices of np.array (which are produced by widely used kernels\n        and are used in most kaggle competitions datasets)\n    '''\n    \n    a = np.fromiter(rle.split(), dtype=np.uint)\n    a = a.reshape((-1, 2))  # an array of (start, length) pairs\n    a[:,0] -= 1  # `start` is 1-indexed\n    \n    y0 = a[:,0] % shape[0]\n    y1 = y0 + a[:,1]\n    if np.any(y1 > shape[0]):\n        # got `y` overrun, meaning that there are a pixels in mask on 0 and shape[0] position\n        y0 = 0\n        y1 = shape[0]\n    else:\n        y0 = np.min(y0)\n        y1 = np.max(y1)\n    \n    x0 = a[:,0] // shape[0]\n    x1 = (a[:,0] + a[:,1]) // shape[0]\n    x0 = np.min(x0)\n    x1 = np.max(x1)\n    \n    if x1 > shape[1]:\n        # just went out of the image dimensions\n        raise ValueError(\"invalid RLE or image dimensions: x1=%d > shape[1]=%d\" % (\n            x1, shape[1]\n        ))\n\n    return x0, y0, x1, y1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get image file path and add it to our dataframe\ndirname = '/kaggle/input/imaterialist-fashion-2020-fgvc7/train/'\ndf_copy = df[:4000].copy()\ndf_copy['ImageId'] = dirname + df_copy['ImageId'] + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get bboxes for each mask\nbboxes = [rle2bbox(c.EncodedPixels, (c.Height, c.Width)) for n, c in df_copy.iterrows()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(bboxes) == df_copy.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bboxes_array = np.array(bboxes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add each coordinate as a column\ndf_copy['x0'], df_copy['y0'], df_copy['x1'], df_copy['y1'] = bboxes_array[:,0], bboxes_array[:,1], bboxes_array[:,2], bboxes_array[:,3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Replace NaNs from AttributeIds by -1\ndf_copy = df_copy.fillna(999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extremely ugly function - need to refactor\ndef attr_str_to_list(df):\n    '''\n    Function that transforms DataFrame AttributeIds which are of type string into a \n    list of integers. Strings must be converted because they cannot be transformed into Tensors\n    '''\n    # cycle through all the non NaN rows - NaN causes an error\n    for index, row in df.iterrows():\n        \n        # Treating str differently than int\n        if isinstance(row['AttributesIds'], str):\n            \n            # Convert each row's string into a list of strings             \n            df['AttributesIds'][index] = row['AttributesIds'].split(',')\n            \n            # Convert each string in the list to int\n            df['AttributesIds'][index] = [int(x) for x in df['AttributesIds'][index]]\n            \n        # If int - make it a list of length 1\n        if isinstance(row['AttributesIds'], int):\n            df['AttributesIds'][index] = [999]\n            \n        # Convert list to array\n        df['AttributesIds'][index] = np.array(df['AttributesIds'][index])\n\n        # Pad array with 0's so that all arrays are the same length - This will allows us to convert to tensor\n        df['AttributesIds'][index] = np.pad(df['AttributesIds'][index], (0, 14-len(df['AttributesIds'][index]))) \n       \nattr_str_to_list(df_copy) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_copy.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del bboxes\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Detectron2 dataset dict (also fetching attributes) "},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://detectron2.readthedocs.io/tutorials/datasets.html\n# https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5\n\nfrom detectron2.structures import BoxMode\nimport pycocotools\n\ndef get_fashion_dict(df):\n    \n    dataset_dicts = []\n    \n    for idx, filename in enumerate(df['ImageId'].unique().tolist()):\n        \n        record = {}\n        \n        # Convert to int otherwise evaluation will throw an error\n        record['height'] = int(df[df['ImageId']==filename]['Height'].values[0])\n        record['width'] = int(df[df['ImageId']==filename]['Width'].values[0])\n        \n        record['file_name'] = filename\n        record['image_id'] = idx\n        \n        objs = []\n        for index, row in df[(df['ImageId']==filename)].iterrows():\n            \n            # Get binary mask\n            mask = rle_decode_string(row['EncodedPixels'], row['Height'], row['Width'])\n            \n            # opencv 4.2+\n            # Transform the mask from binary to polygon format\n            contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE,\n                                                    cv2.CHAIN_APPROX_SIMPLE)\n            \n            # opencv 3.2\n            # mask_new, contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE,\n            #                                            cv2.CHAIN_APPROX_SIMPLE)\n            \n            segmentation = []\n\n            for contour in contours:\n                contour = contour.flatten().tolist()\n                # segmentation.append(contour)\n                if len(contour) > 4:\n                    segmentation.append(contour)    \n            \n            # Data for each mask\n            obj = {\n                'bbox': [row['x0'], row['y0'], row['x1'], row['y1']],\n                'bbox_mode': BoxMode.XYXY_ABS,\n                'category_id': row['ClassId'],\n                'attributes': row['AttributesIds'],\n                'segmentation': segmentation,\n                'iscrowd': 0\n            }\n            objs.append(obj)\n        record['annotations'] = objs\n        dataset_dicts.append(record)\n    return dataset_dicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# To view a sample of fashion_dict\nfashion_dict = get_fashion_dict(df_copy[:50])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fashion_dict[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Attribute holder class "},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import Iterator, List, Tuple, Union\nimport torch.nn.functional as F\n\n# Base Attribute holder\nclass Attributes:\n    \"\"\"\n    This structure stores a list of attributes as a Nx14 torch.Tensor (14 because we added a padding of 14 to all our attributes\n    so that they can have the same length.\n    It behaves like a Tensor\n    (support indexing, `to(device)`, `.device`, `non empty`, and iteration over all attributes)\n    \"\"\"\n    \n    def __init__(self, tensor: torch.Tensor):\n        \"\"\"\n        Args:\n            tensor (Tensor[float]): a Nx13 matrix.  Each row is [attribute_1, attribute_2, ...].\n        \"\"\"\n        device = tensor.device if isinstance(tensor, torch.Tensor) else torch.device(\"cpu\")\n        tensor = torch.as_tensor(tensor, dtype=torch.int64, device=device)\n        assert tensor.dim() == 2, tensor.size()\n\n        self.tensor = tensor\n\n\n    def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> \"Boxes\":\n        \"\"\"\n        Returns:\n            Attributes: Create a new :class:`Attributes` by indexing.\n        The following usage are allowed:\n        1. `new_attributes = attributes[3]`: return a `Attributes` which contains only one Attribute.\n        2. `new_attributes = attributes[2:10]`: return a slice of attributes.\n        3. `new_attributes = attributes[vector]`, where vector is a torch.BoolTensor\n           with `length = len(attributes)`. Nonzero elements in the vector will be selected.\n        Note that the returned Attributes might share storage with this Attributes,\n        subject to Pytorch's indexing semantics.\n        \"\"\"\n        \n        b = self.tensor[item]\n        assert b.dim() == 2, \"Indexing on Attributes with {} failed to return a matrix!\".format(item)\n        return Attributes(b)\n\n    def __len__(self) -> int:\n        return self.tensor.shape[0]\n    \n    def to(self, device: str) -> \"Attributes\":\n        return Attributes(self.tensor.to(device))\n\n\n    def nonempty(self, threshold: float = 0.0) -> torch.Tensor:\n        \"\"\"\n        Find attributes that are non-empty.\n        An attribute is considered empty if its first attribute in the list is 999.\n        Returns:\n            Tensor:\n                a binary vector which represents whether each attribute is empty\n                (False) or non-empty (True).\n        \"\"\"\n        attributes = self.tensor\n        first_attr = attributes[:, 0]\n        keep = (first_attr != 999)\n        return keep\n\n    def __repr__(self) -> str:\n        return \"Attributes(\" + str(self.tensor) + \")\"\n\n\n    def remove_padding(self, attribute):\n        'WIP'\n        pass\n\n    @property\n    def device(self) -> torch.device:\n        return self.tensor.device\n\n    def __iter__(self) -> Iterator[torch.Tensor]:\n        \"\"\"\n        Yield attributes as a Tensor of shape (14,) at a time.\n        \"\"\"\n        yield from self.tensor","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Custom DatasetMapper"},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nfrom detectron2.data import build_detection_train_loader, build_detection_test_loader\nfrom detectron2.data import transforms as T\nfrom detectron2.data import detection_utils as utils\n\n\nclass DatasetMapper:\n    \"\"\"\n    A callable which takes a dataset dict in Detectron2 Dataset format,\n    and map it into a format used by the model.\n\n    This is a custom version of the DatasetMapper. The only different with Detectron2's \n    DatasetMapper is that we extract attributes from our dataset_dict. \n    \"\"\"\n\n    def __init__(self, cfg, is_train=True):\n        if cfg.INPUT.CROP.ENABLED and is_train:\n            self.crop_gen = T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)\n            logging.getLogger(__name__).info(\"CropGen used in training: \" + str(self.crop_gen))\n        else:\n            self.crop_gen = None\n\n        self.tfm_gens = utils.build_transform_gen(cfg, is_train)\n\n        # fmt: off\n        self.img_format     = cfg.INPUT.FORMAT\n        self.mask_on        = cfg.MODEL.MASK_ON\n        self.mask_format    = cfg.INPUT.MASK_FORMAT\n        self.keypoint_on    = cfg.MODEL.KEYPOINT_ON\n        self.load_proposals = cfg.MODEL.LOAD_PROPOSALS\n        # fmt: on\n        if self.keypoint_on and is_train:\n            # Flip only makes sense in training\n            self.keypoint_hflip_indices = utils.create_keypoint_hflip_indices(cfg.DATASETS.TRAIN)\n        else:\n            self.keypoint_hflip_indices = None\n\n        if self.load_proposals:\n            self.min_box_side_len = cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE\n            self.proposal_topk = (\n                cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN\n                if is_train\n                else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST\n            )\n        self.is_train = is_train\n\n    def __call__(self, dataset_dict):\n        \"\"\"\n        Args:\n            dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.\n\n        Returns:\n            dict: a format that builtin models in detectron2 accept\n        \"\"\"\n        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below\n        # USER: Write your own image loading if it's not from a file\n        image = utils.read_image(dataset_dict[\"file_name\"], format=self.img_format)\n        utils.check_image_size(dataset_dict, image)\n\n        if \"annotations\" not in dataset_dict:\n            image, transforms = T.apply_transform_gens(\n                ([self.crop_gen] if self.crop_gen else []) + self.tfm_gens, image\n            )\n        else:\n            # Crop around an instance if there are instances in the image.\n            # USER: Remove if you don't use cropping\n            if self.crop_gen:\n                crop_tfm = utils.gen_crop_transform_with_instance(\n                    self.crop_gen.get_crop_size(image.shape[:2]),\n                    image.shape[:2],\n                    np.random.choice(dataset_dict[\"annotations\"]),\n                )\n                image = crop_tfm.apply_image(image)\n            image, transforms = T.apply_transform_gens(self.tfm_gens, image)\n            if self.crop_gen:\n                transforms = crop_tfm + transforms\n\n        image_shape = image.shape[:2]  # h, w\n\n        # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,\n        # but not efficient on large generic data structures due to the use of pickle & mp.Queue.\n        # Therefore it's important to use torch.Tensor.\n        dataset_dict[\"image\"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1)))\n\n        # USER: Remove if you don't use pre-computed proposals.\n        if self.load_proposals:\n            utils.transform_proposals(\n                dataset_dict, image_shape, transforms, self.min_box_side_len, self.proposal_topk\n            )\n\n        if not self.is_train:\n            # USER: Modify this if you want to keep them for some reason.\n            dataset_dict.pop(\"annotations\", None)\n            dataset_dict.pop(\"sem_seg_file_name\", None)\n            return dataset_dict\n\n        if \"annotations\" in dataset_dict:\n            # USER: Modify this if you want to keep them for some reason.\n            for anno in dataset_dict[\"annotations\"]:\n                if not self.mask_on:\n                    anno.pop(\"segmentation\", None)\n                if not self.keypoint_on:\n                    anno.pop(\"keypoints\", None)\n\n            # USER: Implement additional transformations if you have other types of data\n            annos = [\n                utils.transform_instance_annotations(\n                    obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices\n                )\n                for obj in dataset_dict.pop(\"annotations\")\n                if obj.get(\"iscrowd\", 0) == 0\n            ]\n            instances = utils.annotations_to_instances(\n                annos, image_shape, mask_format=self.mask_format\n            )\n            # Create a tight bounding box from masks, useful when image is cropped\n            if self.crop_gen and instances.has(\"gt_masks\"):\n                instances.gt_boxes = instances.gt_masks.get_bounding_boxes()           \n            \n            ####################################\n            # New: Get attributes from annos \n            ####################################\n            if len(annos) and 'attributes' in annos[0]:\n    \n                # get a list of list of attributes\n                gt_attributes = [x['attributes'] for x in annos]\n                \n                # Put attributes in Attributes class holder and add them to instances\n                instances.gt_attributes = Attributes(gt_attributes)\n                \n            dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n\n        # USER: Remove if you don't do semantic/panoptic segmentation.\n        if \"sem_seg_file_name\" in dataset_dict:\n            with PathManager.open(dataset_dict.pop(\"sem_seg_file_name\"), \"rb\") as f:\n                sem_seg_gt = Image.open(f)\n                sem_seg_gt = np.asarray(sem_seg_gt, dtype=\"uint8\")\n            sem_seg_gt = transforms.apply_segmentation(sem_seg_gt)\n            sem_seg_gt = torch.as_tensor(sem_seg_gt.astype(\"long\"))\n            dataset_dict[\"sem_seg\"] = sem_seg_gt\n        return dataset_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom Trainer"},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\n\nclass FashionTrainer(DefaultTrainer):\n    'A customized version of DefaultTrainer. We add the mapping `DatasetMapper` to the dataloader.'\n    \n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(cfg, mapper=DatasetMapper(cfg))\n    \n    @classmethod\n    def build_test_loader(cls, cfg, dataset_name):\n        return build_detection_test_loader(cfg, dataset_name, mapper=DatasetMapper(cfg))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get a sample of the training data to run experiments\ndf_copy_train = df_copy[:3000].copy()\ndf_copy_test = df_copy[-1000:].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" from detectron2.data import DatasetCatalog, MetadataCatalog\n\n# Register the train set metadata\nfor d in ['train']:\n    DatasetCatalog.register('1sample_fashion_' + d, lambda d=df_copy_train: get_fashion_dict(d))\n    MetadataCatalog.get(\"1sample_fashion_\" + d).set(thing_classes=list(df_categories.name))\nfashion_metadata = MetadataCatalog.get(\"1sample_fashion_train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Register the test and set metadata\nfor d in ['test']:\n    DatasetCatalog.register('sample_fashion_' + d, lambda d=df_copy_test: get_fashion_dict(d))\n    MetadataCatalog.get(\"sample_fashion_\" + d).set(thing_classes=list(df_categories.name))\nfashion_metadata = MetadataCatalog.get(\"sample_fashion_test\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View some images + masks from the dataset\nimport random\nfor d in random.sample(fashion_dict, 3):\n    plt.figure(figsize=(10,10))\n    img = cv2.imread(d[\"file_name\"])\n    visualizer = Visualizer(img[:, :, ::-1], fashion_metadata, scale=0.5)\n    vis = visualizer.draw_dataset_dict(d)\n    plt.imshow(vis.get_image()[:, :, ::-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.engine import DefaultTrainer\nfrom detectron2.config import get_cfg\n\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"1sample_fashion_train\",)\ncfg.DATASETS.TEST = ()\ncfg.DATALOADER.NUM_WORKERS = 1\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\n\n##### Input #####\n# Set a smaller image size than default to avoid memory problems\n\n# Size of the smallest side of the image during training\ncfg.INPUT.MIN_SIZE_TRAIN = (40,)\n# Maximum size of the side of the image during training\ncfg.INPUT.MAX_SIZE_TRAIN = 60\n# Size of the smallest side of the image during testing. Set to zero to disable resize in testing.\ncfg.INPUT.MIN_SIZE_TEST = 40\n# Maximum size of the side of the image during testing\ncfg.INPUT.MAX_SIZE_TEST = 60\n\n# Mask type\n#cfg.INPUT.MASK_FORMAT = \"bitmask\"  # default: \"polygon\"\n\n\ncfg.SOLVER.IMS_PER_BATCH = 1\ncfg.SOLVER.BASE_LR = 0.00025  \ncfg.SOLVER.MAX_ITER = 50 # not enough iterations for real training, but this is just a demo    \ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   # default: 512\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 46  # 46 classes in iMaterialist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainer = DefaultTrainer(cfg) \ntrainer = FashionTrainer(cfg) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.resume_or_load(resume=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we perform inference with the trained model on the sample_fashion_test dataset. First, let's create a predictor using the model we just trained:"},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model\ncfg.DATASETS.TEST = ('sample_fashion_test',)\npredictor = DefaultPredictor(cfg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\nplt.figure(figsize=(20,20))\nfor d in random.sample(fashion_dict, 3):    \n    im = cv2.imread(d[\"file_name\"])\n    outputs = predictor(im)\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=fashion_metadata, \n                   scale=0.8, \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n    )\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(vis.get_image()[:, :, ::-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show different images at random\nrows, cols = 3, 3\nplt.figure(figsize=(20,20))\n\nfor i, d in enumerate(random.sample(fashion_dict, 9)):\n    \n    # Process image\n    plt.subplot(rows, cols, i+1)\n\n    im = cv2.imread(d[\"file_name\"])\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    \n    # Run through predictor\n    outputs = predictor(im)\n    \n    # Visualize\n    v = Visualizer(im[:, :, ::-1],\n                   metadata=fashion_metadata, \n                   scale=0.8, \n                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n    )\n    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(v.get_image()[:, :, ::-1])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"WIP"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}