{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import torch \nimport torch.nn as nn ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ImageFile\nimport numpy as np \nimport os\nimport glob \nimport matplotlib.pyplot as plt \nimport matplotlib.image as mpimg\nimport pandas as pd \nimport json\nimport gc\nfrom pathlib import Path\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloader tutorial \n* [How to show PIL Image in ipython notebook](https://stackoverflow.com/questions/26649716/how-to-show-pil-image-in-ipython-notebook)\n* [Dataloader for semantic segmentation](https://discuss.pytorch.org/t/dataloader-for-semantic-segmentation/48290)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = \"/kaggle/input/imaterialist-fashion-2020-fgvc7/train\"\nimg_files = glob.glob(os.path.join(train_dir, \"*.jpg\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 0\npil_im = Image.open(img_files[idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline \nprint(\"file_name: \", img_files[idx])\nplt.imshow(np.asarray(pil_im))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Get Training and Validation Data Loader\n* [iMaterialist Detectron2](https://www.kaggle.com/julienbeaulieu/imaterialist-detectron2)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = Path(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/\")\nimg_dir =  Path(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/train\")\ntrain_data = pd.read_csv(data_dir/\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _Start: to get label descriptions \nwith open(data_dir/\"label_descriptions.json\", 'r') as file:\n    label_desc = json.load(file)\n# _End: to get label descriptions ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_desc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _Start: Classes and Attributes processing \ndf_categories = pd.DataFrame(label_desc['categories'])\ndf_attributes = pd.DataFrame(label_desc['attributes'])\n# _End: Classes and Attributes processing \n\nn_classes = len(label_desc['categories'])\nn_attributes = len(label_desc['attributes'])\n\nprint('Classes: {0} \\nAttributes: {1}'.\n     format(str(n_classes), str(n_attributes)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_categories","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_attributes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_categories.supercategory.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_attributes.supercategory.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check out training images and their masks\n* [iMaterialist - Segmentation task](https://www.kaggle.com/yanastamenova/imaterialist-segmentation-task)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images(size = 4, figsize = (12, 12)):\n    #get the images\n    image_ids = train_data['ImageId'].unique()[:size]\n    images = []\n    \n    for image_id in image_ids:\n        images.append(mpimg.imread('{0}/train/{1}.jpg'.format(data_dir, image_id)))\n        \n    count = 0\n    \n    fig, ax = plt.subplots(nrows = 2, ncols = 2, figsize = figsize)\n    for row in ax:\n        for col in row:\n            col.imshow(images[count])\n            col.axis('off')\n            count += 1\n    plt.show()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_images()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Function to create mask\ndef create_mask(size):\n    image_ids = train_data['ImageId'].unique()[:size] #get a number of images\n    images_meta = [] #to be added in this array\n    \n    for image_id in image_ids:\n        img = mpimg.imread('{0}/train/{1}.jpg'.format(data_dir, image_id))\n        images_meta.append({\n            'image': img,\n            'shape': img.shape,\n            'encoded_pixels': train_data[train_data['ImageId'] == image_id]['EncodedPixels'],\n            'class_ids': train_data[train_data['ImageId'] == image_id]['ClassId']\n        })\n        \n    masks = []\n    \n    for image in images_meta:\n        shape = image.get('shape') #get via key\n        encoded_pixels = list(image.get('encoded_pixels')) \n        class_ids = list(image.get('class_ids'))\n        \n        #Initialize numpy array with shape same as image size\n        height, width = shape[:2] \n        mask = np.zeros((height, width)).reshape(-1) \n        # (-1) 'The new shape should be compatible with the original shape'\n        # numpy allow us to give one of new shape parameter as -1 but not (-1, -1)).\n        # It means that it is an unknown dimension and we want numpy to figure it out.\n        # And numpy will figure this by looking at the 'length of the array and remaining\n        # dimensions' and making sure it satisfies the above mentioned criteria\n        \n        #Iterate over encoded pixels and create mask\n        for segment, (pixel_str, class_id) in enumerate(zip(encoded_pixels, class_ids)):\n            splitted_pixels = list(map(int, pixel_str.split()))      #split the pixels string\n            pixel_starts = splitted_pixels[::2]                      #choose every second element\n            run_lengths = splitted_pixels[1::2]                      #start from 1 with step size 2\n            assert max(pixel_starts) < mask.shape[0]                 #make sure it is ok\n            \n            for pixel_start, run_length in zip(pixel_starts, run_lengths):\n                pixel_start = int(pixel_start) - 1\n                run_length = int(run_length)\n                mask[pixel_start:pixel_start+run_length] = 255 - class_id \n        masks.append(mask.reshape((height, width), order = 'F'))\n    \n    return masks, images_meta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_segmented_images(size = 4, figsize = (14, 14)):\n    #First, create masks from given segments\n    masks, images_meta = create_mask(size)\n    \n    #Plot images\n    \n    count = 0\n    \n    fig, ax = plt.subplots(nrows = 2, ncols = 2, figsize = figsize)\n    for row in ax:\n        for col in row:\n            col.imshow(images_meta[count]['image'])\n            col.imshow(masks[count], alpha = 0.50)\n            col.axis('off')\n            count += 1\n    plt.show()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_segmented_images()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Grouping data based on each identical image\n* refer to [here](https://www.kaggle.com/yanastamenova/imaterialist-segmentation-task)\n* Use the ```pandas``` for grouping proces \n* [Custom Dataset format for PyTorch](https://wikidocs.net/57165)\n* [Building Efficient Custom Datasets in PyTorch](https://towardsdatascience.com/building-efficient-custom-datasets-in-pytorch-2563b946fd9f)\n* [fashionChallenge-SemSegmentation/dataHandling.py](https://github.com/mcreduardo/fashionChallenge-SemSegmentation/blob/master/dataHandling.py)\n* [Torchvision Object Detection](https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html#torchvision-object-detection-finetuning-tutorial)\n* [Pytorch, Custom Dataset, YouTube](https://www.youtube.com/watch?v=38hn-FpRaJs)\n* [IMaterialist 2020: Starter EDA + Mask RCNN](https://www.kaggle.com/kaushal2896/imaterialist-2020-starter-eda-mask-rcnn)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"images_data = train_data.groupby('ImageId')['EncodedPixels', 'ClassId'].agg(lambda x: list(x))\ndimensions_data = train_data.groupby('ImageId')['Height', 'Width'].mean()\n\nimages_data = images_data.join(dimensions_data, on='ImageId')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_data.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total images: \", len(images_data))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataLoader for Pytorch \n* [Torchvision Object Detection](https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html#torchvision-object-detection-finetuning-tutorial)\n* [IMaterialist 2020: Starter EDA + Mask RCNN](https://www.kaggle.com/kaushal2896/imaterialist-2020-starter-eda-mask-rcnn)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fashion2020dataset(torch.utils.data.Dataset):\n    def __init__(self, data_root, transforms, df_csv:str ):  # _to load and preprocess for the dataset. \n        \n        super().__init__()\n        self.data_root = data_root         \n        self.transforms = transforms \n        self.imgs = list(sorted(os.listdir(os.path.join(data_root, \"train\"))))\n        \n        # _Start: read .csv with pandas for DataFormat description \n        self.df_csv = pd.read_csv(os.path.join(data_root, df_csv))  \n        self.image_ids = self.df_csv[\"ImageId\"].unique() # to get all image names\n        \n        \n         \n    def __getitem__(self, idx): # _to get a specific item.                        \n        \n        imgID = self.image_ids[idx]       \n        #imgID = self.imgs[idx]       \n        \n        print(f\"Image loading: {imgID}\")\n        \n        pil_im = Image.open(\"{0}/train/{1}.jpg\".format(self.data_root, str(imgID)))\n        img = np.asarray(pil_im)\n        \n        images_meta = {} # \n        images_meta.update({ \"image\":img,\n                             \"shape\":img.shape, \n                             \"encoded_pixels\": self.df_csv[self.df_csv['ImageId'] == imgID]['EncodedPixels'],\n                             \"class_ids\" : self.df_csv[self.df_csv['ImageId'] == imgID]['ClassId']                                   \n                             })\n            \n        # _Start: create masks with decoding and bbox from them \n        masks = []     \n        boxes = [] \n\n        shape = images_meta.get(\"shape\")  # _get via key of dict() \n        encoded_pixels = list(images_meta.get(\"encoded_pixels\"))\n        class_ids = list(images_meta.get(\"class_ids\"))\n        print(class_ids)\n            \n        # _Initialze an empty array with the same shape as the image \n        height, width = shape[:2] \n        mask = np.zeros((height, width)).reshape(-1)\n        # (-1) 'The new shape should be compatible with the original shape'\n            \n        pbarLoad = tqdm(zip(encoded_pixels, class_ids))\n        for segment, (pixel_str, class_id) in enumerate(pbarLoad):\n            pbarLoad.set_description(f\"Loading encoded pixels...: {segment}\" )\n            splitted_pixels = list(map(int, pixel_str.split())) #split the pixels string\n            pixel_starts = splitted_pixels[::2] #choose every second element\n            run_lengths = splitted_pixels[1::2]  #start from 1 with step size 2\n               \n            assert max(pixel_starts) < mask.shape[0]  \n            \n            pbarDecode = tqdm(zip(pixel_starts, run_lengths))    \n            for pixel_start, run_length in pbarDecode:\n                pbarDecode.set_description(f\"Decoding masks...: {pixel_start}\" )\n                pixel_start = int(pixel_start) - 1\n                run_length = int(run_length)\n                mask[pixel_start:pixel_start+run_length] = 255 - class_id *4\n                         \n            \n            mask = mask.reshape((height, width), order = 'F')\n            masks.append(mask)\n            \n            # _Start: get bounding box coordinates from each mask \n            pos = np.where(mask)\n            xmin = np.min(pos[1])\n            xmax = np.max(pos[1])\n            ymin = np.min(pos[0])\n            ymax = np.max(pos[0])\n            boxes.append([xmin, ymin, xmax, ymax])\n            # _End: get bounding box coordinates from each mask \n            \n            mask = np.zeros((height, width)).reshape(-1) # re-initialize \n        # _End: create masks with decoding \n        \n        \n        \n        # _Start: convert everything into a torch.Tensor \n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        class_ids = torch.as_tensor(class_ids, dtype=torch.uint8)\n        masks = torch.as_tensor(masks, dtype=torch.uint8)  \n        image_id = torch.tensor([idx])\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])      \n        \n        iscrowd = torch.zeros(len(class_ids,), dtype=torch.int64) # suppose all instances are not crowd\n        # _End: convert everything into a torch.Tensor\n        \n        \n        target = {}\n        target[\"boxes\"] = boxes\n        target[\"class_ids\"] = class_ids\n        target[\"masks\"] = masks\n        target[\"image_id\"] = image_id \n        target[\"area\"] = area\n        target[\"iscrowd\"] = iscrowd           \n        \n        \n        if self.transforms is not None: \n            img, target = self.transforms(img, target)           \n        \n        return img, target\n    \n    def __len__(self):          # _to return the length of data samples in the dataset. \n        return len(self.imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Dataloader = Fashion2020dataset(data_root= data_dir, transforms=None, df_csv=\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, target = Dataloader.__getitem__(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target[\"image_id\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target[\"iscrowd\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(target[\"masks\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(target[\"masks\"][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target[\"class_ids\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target[\"boxes\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display the processed data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.patches import Rectangle\n\n%matplotlib inline \nidx = 5      # _ change the number withing range \nxmin, ymin, xmax, ymax = target[\"boxes\"][idx]\n\nplt.imshow(img)\nplt.imshow(target[\"masks\"][idx], alpha=0.7)\n\nplt.gca().add_patch(Rectangle((xmin,ymin),xmax-xmin  , ymax-ymin, linewidth=1,edgecolor='r',facecolor='none'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.patches import Rectangle\n\n%matplotlib inline \nidx = 3      # _ change the number withing range \nxmin, ymin, xmax, ymax = target[\"boxes\"][idx]\n\nplt.imshow(img)\nplt.imshow(target[\"masks\"][idx], alpha=0.7)\n\nplt.gca().add_patch(Rectangle((xmin,ymin),xmax-xmin  , ymax-ymin, linewidth=1,edgecolor='r',facecolor='none'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}