{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import required libraries\nimport os\nimport gc\nimport sys\nimport json\nimport random\nfrom pathlib import Path\n\nimport cv2 # CV2 for image manipulation\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\n\nimport seaborn as sns\nimport matplotlib.image as mpimg\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold, KFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-23T16:59:58.244804Z","iopub.execute_input":"2022-04-23T16:59:58.24563Z","iopub.status.idle":"2022-04-23T16:59:58.25669Z","shell.execute_reply.started":"2022-04-23T16:59:58.245579Z","shell.execute_reply":"2022-04-23T16:59:58.254358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow==1.5\n!pip install keras==2.1.5\n\nimport tensorflow\nprint(tensorflow.__version__)\nimport keras\nprint(keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T16:59:58.259058Z","iopub.execute_input":"2022-04-23T16:59:58.259792Z","iopub.status.idle":"2022-04-23T17:00:13.077231Z","shell.execute_reply.started":"2022-04-23T16:59:58.259708Z","shell.execute_reply":"2022-04-23T17:00:13.074906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith open('/kaggle/input/imaterialist-fashion-2020-fgvc7/label_descriptions.json', 'r') as file:\n    label_desc = json.load(file)\nsample_sub_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2020-fgvc7/sample_submission.csv')\ntrain_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2020-fgvc7/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:13.080868Z","iopub.execute_input":"2022-04-23T17:00:13.081218Z","iopub.status.idle":"2022-04-23T17:00:25.25835Z","shell.execute_reply.started":"2022-04-23T17:00:13.081176Z","shell.execute_reply":"2022-04-23T17:00:25.256785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.261515Z","iopub.execute_input":"2022-04-23T17:00:25.261862Z","iopub.status.idle":"2022-04-23T17:00:25.279993Z","shell.execute_reply.started":"2022-04-23T17:00:25.261825Z","shell.execute_reply":"2022-04-23T17:00:25.278374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.281722Z","iopub.execute_input":"2022-04-23T17:00:25.282059Z","iopub.status.idle":"2022-04-23T17:00:25.304188Z","shell.execute_reply.started":"2022-04-23T17:00:25.282025Z","shell.execute_reply":"2022-04-23T17:00:25.302647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Shape of training dataset: {train_df.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.306257Z","iopub.execute_input":"2022-04-23T17:00:25.306596Z","iopub.status.idle":"2022-04-23T17:00:25.3207Z","shell.execute_reply.started":"2022-04-23T17:00:25.306555Z","shell.execute_reply":"2022-04-23T17:00:25.318977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'# of images in training set: {train_df[\"ImageId\"].nunique()}')\nprint(f'# of images in test set: {sample_sub_df[\"ImageId\"].nunique()}')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.32211Z","iopub.execute_input":"2022-04-23T17:00:25.322476Z","iopub.status.idle":"2022-04-23T17:00:25.395871Z","shell.execute_reply.started":"2022-04-23T17:00:25.322434Z","shell.execute_reply":"2022-04-23T17:00:25.394121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image size analysis in training dataset","metadata":{}},{"cell_type":"code","source":"pd.DataFrame([train_df['Height'].describe(), train_df['Width'].describe()]).T.loc[['max', 'min', 'mean']]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.39756Z","iopub.execute_input":"2022-04-23T17:00:25.398155Z","iopub.status.idle":"2022-04-23T17:00:25.427504Z","shell.execute_reply.started":"2022-04-23T17:00:25.398109Z","shell.execute_reply":"2022-04-23T17:00:25.426452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Height and Width destribution of training images","metadata":{}},{"cell_type":"code","source":"image_shape_df = train_df.groupby(\"ImageId\")[\"Height\", \"Width\"].first()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.428666Z","iopub.execute_input":"2022-04-23T17:00:25.429058Z","iopub.status.idle":"2022-04-23T17:00:25.493906Z","shell.execute_reply.started":"2022-04-23T17:00:25.429021Z","shell.execute_reply":"2022-04-23T17:00:25.493006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 5))\nax1.hist(image_shape_df['Height'], bins=100)\nax1.set_title(\"Height distribution\")\nax2.hist(image_shape_df['Width'], bins=100)\nax2.set_title(\"Width distribution\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:25.495411Z","iopub.execute_input":"2022-04-23T17:00:25.495632Z","iopub.status.idle":"2022-04-23T17:00:26.280896Z","shell.execute_reply.started":"2022-04-23T17:00:25.495605Z","shell.execute_reply":"2022-04-23T17:00:26.279441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with minimum height","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nmin_height = list(set(train_df[train_df['Height'] == train_df['Height'].min()]['ImageId']))[0]\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{min_height}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:26.282734Z","iopub.execute_input":"2022-04-23T17:00:26.283036Z","iopub.status.idle":"2022-04-23T17:00:26.653669Z","shell.execute_reply.started":"2022-04-23T17:00:26.283Z","shell.execute_reply":"2022-04-23T17:00:26.652388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with maximum height","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nmax_height = list(set(train_df[train_df['Height'] == train_df['Height'].max()]['ImageId']))[0]\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{max_height}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:26.659373Z","iopub.execute_input":"2022-04-23T17:00:26.659735Z","iopub.status.idle":"2022-04-23T17:00:35.342182Z","shell.execute_reply.started":"2022-04-23T17:00:26.659696Z","shell.execute_reply":"2022-04-23T17:00:35.340876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with minimum width","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nmin_width = list(set(train_df[train_df['Width'] == train_df['Width'].min()]['ImageId']))[0]\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{min_width}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:35.348247Z","iopub.execute_input":"2022-04-23T17:00:35.348651Z","iopub.status.idle":"2022-04-23T17:00:35.684931Z","shell.execute_reply.started":"2022-04-23T17:00:35.348604Z","shell.execute_reply":"2022-04-23T17:00:35.68218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with maximum width","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nmax_width = list(set(train_df[train_df['Width'] == train_df['Width'].max()]['ImageId']))[0]\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{max_width}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:35.687222Z","iopub.execute_input":"2022-04-23T17:00:35.687594Z","iopub.status.idle":"2022-04-23T17:00:42.755019Z","shell.execute_reply.started":"2022-04-23T17:00:35.687548Z","shell.execute_reply":"2022-04-23T17:00:42.753638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"area_df = pd.DataFrame()\narea_df['ImageId'] = train_df['ImageId']\narea_df['area'] = train_df['Height'] * train_df['Width']\nmin_area = list(set(area_df[area_df['area'] == area_df['area'].min()]['ImageId']))[0]\nmax_area = list(set(area_df[area_df['area'] == area_df['area'].max()]['ImageId']))[0]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:42.756432Z","iopub.execute_input":"2022-04-23T17:00:42.756681Z","iopub.status.idle":"2022-04-23T17:00:42.801395Z","shell.execute_reply.started":"2022-04-23T17:00:42.756653Z","shell.execute_reply":"2022-04-23T17:00:42.800028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with minimum area","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{min_area}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:42.802713Z","iopub.execute_input":"2022-04-23T17:00:42.802968Z","iopub.status.idle":"2022-04-23T17:00:43.124463Z","shell.execute_reply.started":"2022-04-23T17:00:42.80294Z","shell.execute_reply":"2022-04-23T17:00:43.123318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image with maximum area","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (70,7))\nplt.imshow(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{max_area}.jpg'))\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:43.125672Z","iopub.execute_input":"2022-04-23T17:00:43.125925Z","iopub.status.idle":"2022-04-23T17:00:49.465468Z","shell.execute_reply.started":"2022-04-23T17:00:43.125898Z","shell.execute_reply":"2022-04-23T17:00:49.463966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Details about Classes and Attributes","metadata":{}},{"cell_type":"code","source":"num_classes = len(label_desc['categories'])\nnum_attributes = len(label_desc['attributes'])\nprint(f'Total # of classes: {num_classes}')\nprint(f'Total # of attributes: {num_attributes}')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.467016Z","iopub.execute_input":"2022-04-23T17:00:49.467286Z","iopub.status.idle":"2022-04-23T17:00:49.473505Z","shell.execute_reply.started":"2022-04-23T17:00:49.467253Z","shell.execute_reply":"2022-04-23T17:00:49.472479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories_df = pd.DataFrame(label_desc['categories'])\nattributes_df = pd.DataFrame(label_desc['attributes'])\ncategories_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.476286Z","iopub.execute_input":"2022-04-23T17:00:49.476785Z","iopub.status.idle":"2022-04-23T17:00:49.509557Z","shell.execute_reply.started":"2022-04-23T17:00:49.4767Z","shell.execute_reply":"2022-04-23T17:00:49.508519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 300)\nattributes_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.510934Z","iopub.execute_input":"2022-04-23T17:00:49.511178Z","iopub.status.idle":"2022-04-23T17:00:49.563603Z","shell.execute_reply.started":"2022-04-23T17:00:49.511149Z","shell.execute_reply":"2022-04-23T17:00:49.562536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting a few training images without any masks","metadata":{}},{"cell_type":"code","source":"def plot_images(size=12, figsize=(12, 12)):\n    # First get some images to be plotted\n    image_ids = train_df['ImageId'].unique()[:12]\n    images=[]\n    \n    for image in image_ids:\n        images.append(mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{image}.jpg'))\n    \n    # Plot images in groups of 4 images\n    n_groups = 4\n    \n    count = 0\n    for index in range(size // 4):\n        fig, ax = plt.subplots(nrows=2, ncols=2, figsize=figsize)\n        for row in ax:\n            for col in row:\n                col.imshow(images[count])\n                col.axis('off')\n                count += 1\n        plt.show()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.564595Z","iopub.execute_input":"2022-04-23T17:00:49.564816Z","iopub.status.idle":"2022-04-23T17:00:49.572492Z","shell.execute_reply.started":"2022-04-23T17:00:49.564786Z","shell.execute_reply":"2022-04-23T17:00:49.57148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_images()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.573472Z","iopub.execute_input":"2022-04-23T17:00:49.57367Z","iopub.status.idle":"2022-04-23T17:00:49.587124Z","shell.execute_reply.started":"2022-04-23T17:00:49.573642Z","shell.execute_reply":"2022-04-23T17:00:49.58592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting a few images with given segments","metadata":{}},{"cell_type":"code","source":"def create_mask(size):\n    image_ids = train_df['ImageId'].unique()[:size]\n    images_meta=[]\n\n    for image_id in image_ids:\n        img = mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{image_id}.jpg')\n        images_meta.append({\n            'image': img,\n            'shape': img.shape,\n            'encoded_pixels': train_df[train_df['ImageId'] == image_id]['EncodedPixels'],\n            'class_ids':  train_df[train_df['ImageId'] == image_id]['ClassId']\n        })\n\n    masks = []\n    for image in images_meta:\n        shape = image.get('shape')\n        encoded_pixels = list(image.get('encoded_pixels'))\n        class_ids = list(image.get('class_ids'))\n        \n        # Initialize numpy array with shape same as image size\n        height, width = shape[:2]\n        mask = np.zeros((height, width)).reshape(-1)\n        \n        # Iterate over encoded pixels and create mask\n        for segment, (pixel_str, class_id) in enumerate(zip(encoded_pixels, class_ids)):\n            splitted_pixels = list(map(int, pixel_str.split()))\n            pixel_starts = splitted_pixels[::2]\n            run_lengths = splitted_pixels[1::2]\n            assert max(pixel_starts) < mask.shape[0]\n            for pixel_start, run_length in zip(pixel_starts, run_lengths):\n                pixel_start = int(pixel_start) - 1\n                run_length = int(run_length)\n                mask[pixel_start:pixel_start+run_length] = 255 - class_id * 4\n        masks.append(mask.reshape((height, width), order='F'))  # https://stackoverflow.com/questions/45973722/how-does-numpy-reshape-with-order-f-work\n    return masks, images_meta","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.588192Z","iopub.execute_input":"2022-04-23T17:00:49.588401Z","iopub.status.idle":"2022-04-23T17:00:49.60183Z","shell.execute_reply.started":"2022-04-23T17:00:49.588373Z","shell.execute_reply":"2022-04-23T17:00:49.600809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_segmented_images(size=12, figsize=(14, 14)):\n    # First create masks from given segments\n    masks, images_meta = create_mask(size)\n    \n    # Plot images in groups of 4 images\n    n_groups = 4\n    \n    count = 0\n    for index in range(size // 4):\n        fig, ax = plt.subplots(nrows=2, ncols=2, figsize=figsize)\n        for row in ax:\n            for col in row:\n                col.imshow(images_meta[count]['image'])\n                col.imshow(masks[count], alpha=0.75)\n                col.axis('off')\n                count += 1\n        plt.show()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.602908Z","iopub.execute_input":"2022-04-23T17:00:49.603159Z","iopub.status.idle":"2022-04-23T17:00:49.620641Z","shell.execute_reply.started":"2022-04-23T17:00:49.603127Z","shell.execute_reply":"2022-04-23T17:00:49.619658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot_segmented_images()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.621911Z","iopub.execute_input":"2022-04-23T17:00:49.622291Z","iopub.status.idle":"2022-04-23T17:00:49.64048Z","shell.execute_reply.started":"2022-04-23T17:00:49.622234Z","shell.execute_reply":"2022-04-23T17:00:49.639334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysing Categories and Attributes","metadata":{}},{"cell_type":"code","source":"categories_df = pd.DataFrame(label_desc.get('categories'))\nattributes_df = pd.DataFrame(label_desc.get('attributes'))","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.641784Z","iopub.execute_input":"2022-04-23T17:00:49.642167Z","iopub.status.idle":"2022-04-23T17:00:49.655935Z","shell.execute_reply.started":"2022-04-23T17:00:49.642127Z","shell.execute_reply":"2022-04-23T17:00:49.655025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'# of categories: {len(categories_df)}')\nprint(f'# of attributes: {len(attributes_df)}')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.65701Z","iopub.execute_input":"2022-04-23T17:00:49.657341Z","iopub.status.idle":"2022-04-23T17:00:49.669558Z","shell.execute_reply.started":"2022-04-23T17:00:49.65731Z","shell.execute_reply":"2022-04-23T17:00:49.668901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So there are 46 categories (classes) and 294 attributes. Let's see some of the categories and attributes","metadata":{}},{"cell_type":"code","source":"categories_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.670881Z","iopub.execute_input":"2022-04-23T17:00:49.671275Z","iopub.status.idle":"2022-04-23T17:00:49.688455Z","shell.execute_reply.started":"2022-04-23T17:00:49.671241Z","shell.execute_reply":"2022-04-23T17:00:49.68774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attributes_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.68949Z","iopub.execute_input":"2022-04-23T17:00:49.689851Z","iopub.status.idle":"2022-04-23T17:00:49.701412Z","shell.execute_reply.started":"2022-04-23T17:00:49.689816Z","shell.execute_reply":"2022-04-23T17:00:49.700814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_map, attribute_map = {}, {}\nfor cat in label_desc.get('categories'):\n    category_map[cat.get('id')] = cat.get('name')\nfor attr in label_desc.get('attributes'):\n    attribute_map[attr.get('id')] = attr.get('name')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.702502Z","iopub.execute_input":"2022-04-23T17:00:49.702849Z","iopub.status.idle":"2022-04-23T17:00:49.70925Z","shell.execute_reply.started":"2022-04-23T17:00:49.702817Z","shell.execute_reply":"2022-04-23T17:00:49.707883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ClassId'] = train_df['ClassId'].map(category_map)\ntrain_df['ClassId'] = train_df['ClassId'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.710716Z","iopub.execute_input":"2022-04-23T17:00:49.710992Z","iopub.status.idle":"2022-04-23T17:00:49.758439Z","shell.execute_reply.started":"2022-04-23T17:00:49.710959Z","shell.execute_reply":"2022-04-23T17:00:49.757117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's see the class wise distribution of segments in training dataset","metadata":{}},{"cell_type":"code","source":"sns.set(style='darkgrid')\nfig, ax = plt.subplots(figsize = (10,10))\nsns.countplot(y='ClassId',data=train_df , ax=ax, order = train_df['ClassId'].value_counts().index)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:49.759782Z","iopub.execute_input":"2022-04-23T17:00:49.759988Z","iopub.status.idle":"2022-04-23T17:00:50.525162Z","shell.execute_reply.started":"2022-04-23T17:00:49.759963Z","shell.execute_reply":"2022-04-23T17:00:50.523545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now let's visualize an image with all its classes and attributes","metadata":{}},{"cell_type":"code","source":"IMAGE_ID = '000b3ec2c6eaffb491a5abb72c2e3e26'","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:50.527646Z","iopub.execute_input":"2022-04-23T17:00:50.52806Z","iopub.status.idle":"2022-04-23T17:00:50.535085Z","shell.execute_reply.started":"2022-04-23T17:00:50.528014Z","shell.execute_reply":"2022-04-23T17:00:50.533324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the an image id given in the training set for visualization\nvis_df = train_df[train_df['ImageId'] == IMAGE_ID]\nvis_df['ClassId'] = vis_df['ClassId'].cat.codes\nvis_df = vis_df.reset_index(drop=True)\nvis_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:50.537142Z","iopub.execute_input":"2022-04-23T17:00:50.53754Z","iopub.status.idle":"2022-04-23T17:00:50.580718Z","shell.execute_reply.started":"2022-04-23T17:00:50.537496Z","shell.execute_reply":"2022-04-23T17:00:50.579342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above table, this image has 8 segmentes and a few attributes. Let's visualize all of them!","metadata":{}},{"cell_type":"markdown","source":"## Let's first the plot the plain image","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (110,11))\nimage = mpimg.imread(f'/kaggle/input/imaterialist-fashion-2020-fgvc7/train/{IMAGE_ID}.jpg')\nplt.grid(False)\nplt.imshow(image)\nplt.plot()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:50.582062Z","iopub.execute_input":"2022-04-23T17:00:50.582451Z","iopub.status.idle":"2022-04-23T17:00:51.738573Z","shell.execute_reply.started":"2022-04-23T17:00:50.582414Z","shell.execute_reply":"2022-04-23T17:00:51.736669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['ImageId'] == IMAGE_ID]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:51.740843Z","iopub.execute_input":"2022-04-23T17:00:51.74121Z","iopub.status.idle":"2022-04-23T17:00:51.813828Z","shell.execute_reply.started":"2022-04-23T17:00:51.741171Z","shell.execute_reply":"2022-04-23T17:00:51.811809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now let's plot each segment in a separate image","metadata":{}},{"cell_type":"code","source":"segments = list(vis_df['EncodedPixels'])\nclass_ids = list(vis_df['ClassId'])\nmasks = []\nfor segment, class_id in zip(segments, class_ids):\n    \n    height = vis_df['Height'][0]\n    width = vis_df['Width'][0]\n    # Initialize empty mask\n    mask = np.zeros((height, width)).reshape(-1)\n    \n    # Iterate over encoded pixels and create mask\n    splitted_pixels = list(map(int, segment.split()))\n    pixel_starts = splitted_pixels[::2]\n    run_lengths = splitted_pixels[1::2]\n    assert max(pixel_starts) < mask.shape[0]\n    for pixel_start, run_length in zip(pixel_starts, run_lengths):\n        pixel_start = int(pixel_start) - 1\n        run_length = int(run_length)\n        mask[pixel_start:pixel_start+run_length] = 255 - class_id * 4\n\n    mask = mask.reshape((height, width), order='F')\n    masks.append(mask)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:51.816175Z","iopub.execute_input":"2022-04-23T17:00:51.81686Z","iopub.status.idle":"2022-04-23T17:00:51.859597Z","shell.execute_reply.started":"2022-04-23T17:00:51.816808Z","shell.execute_reply":"2022-04-23T17:00:51.858199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_individual_segment(*masks, image, figsize=(110, 11)):\n    plt.figure(figsize = figsize)\n    plt.imshow(image)\n    for mask in masks:\n        plt.imshow(mask, alpha=0.6)\n    plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:51.861217Z","iopub.execute_input":"2022-04-23T17:00:51.861708Z","iopub.status.idle":"2022-04-23T17:00:51.870178Z","shell.execute_reply.started":"2022-04-23T17:00:51.861664Z","shell.execute_reply":"2022-04-23T17:00:51.868175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some of the segments have no attributes. Let's check how many such segment exists in training dataset.","metadata":{}},{"cell_type":"code","source":"print(f'Segments that do not have attributes: {train_df[\"AttributesIds\"].isna().sum()/len(train_df) * 100} %')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:51.871901Z","iopub.execute_input":"2022-04-23T17:00:51.872501Z","iopub.status.idle":"2022-04-23T17:00:51.925543Z","shell.execute_reply.started":"2022-04-23T17:00:51.872446Z","shell.execute_reply":"2022-04-23T17:00:51.924552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check of missing values in training dataset for columns other than \"AttributeIds\"","metadata":{}},{"cell_type":"code","source":"train_df[['ImageId', 'EncodedPixels', 'Height', 'Width', 'ClassId']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:51.927225Z","iopub.execute_input":"2022-04-23T17:00:51.927716Z","iopub.status.idle":"2022-04-23T17:00:52.064483Z","shell.execute_reply.started":"2022-04-23T17:00:51.92767Z","shell.execute_reply":"2022-04-23T17:00:52.063597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation and modeling","metadata":{}},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:52.065918Z","iopub.execute_input":"2022-04-23T17:00:52.066461Z","iopub.status.idle":"2022-04-23T17:00:52.099414Z","shell.execute_reply.started":"2022-04-23T17:00:52.066422Z","shell.execute_reply":"2022-04-23T17:00:52.097638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ClassId'] = train_df['ClassId'].cat.codes","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:52.101599Z","iopub.execute_input":"2022-04-23T17:00:52.103022Z","iopub.status.idle":"2022-04-23T17:00:52.114792Z","shell.execute_reply.started":"2022-04-23T17:00:52.102934Z","shell.execute_reply":"2022-04-23T17:00:52.113845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:52.116249Z","iopub.execute_input":"2022-04-23T17:00:52.116708Z","iopub.status.idle":"2022-04-23T17:00:52.142404Z","shell.execute_reply.started":"2022-04-23T17:00:52.116666Z","shell.execute_reply":"2022-04-23T17:00:52.141573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop attributeIds for simplicity for now. TODO: Need to take this in consideration once the basic model is ready with ClassId","metadata":{}},{"cell_type":"code","source":"train_df = train_df.drop('AttributesIds', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:52.143856Z","iopub.execute_input":"2022-04-23T17:00:52.144428Z","iopub.status.idle":"2022-04-23T17:00:52.170511Z","shell.execute_reply.started":"2022-04-23T17:00:52.144386Z","shell.execute_reply":"2022-04-23T17:00:52.169492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = train_df.groupby('ImageId')['EncodedPixels', 'ClassId'].agg(lambda x: list(x))\nsize_df = train_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:00:52.172991Z","iopub.execute_input":"2022-04-23T17:00:52.173438Z","iopub.status.idle":"2022-04-23T17:01:04.198485Z","shell.execute_reply.started":"2022-04-23T17:00:52.173387Z","shell.execute_reply":"2022-04-23T17:01:04.19629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reference: https://github.com/matterport/Mask_RCNN/blob/master/samples/shapes/train_shapes.ipynb","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:04.200546Z","iopub.execute_input":"2022-04-23T17:01:04.200903Z","iopub.status.idle":"2022-04-23T17:01:09.222505Z","shell.execute_reply.started":"2022-04-23T17:01:04.200857Z","shell.execute_reply":"2022-04-23T17:01:09.220566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:09.225517Z","iopub.execute_input":"2022-04-23T17:01:09.226094Z","iopub.status.idle":"2022-04-23T17:01:13.07473Z","shell.execute_reply.started":"2022-04-23T17:01:09.226039Z","shell.execute_reply":"2022-04-23T17:01:13.073484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cat /kaggle/working/Mask_RCNN/mrcnn/model.py","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.07678Z","iopub.execute_input":"2022-04-23T17:01:13.077148Z","iopub.status.idle":"2022-04-23T17:01:13.082133Z","shell.execute_reply.started":"2022-04-23T17:01:13.077097Z","shell.execute_reply":"2022-04-23T17:01:13.081045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/imaterialist-fashion-2020-fgvc7')\nROOT_DIR = Path('/kaggle/working')","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.083515Z","iopub.execute_input":"2022-04-23T17:01:13.083939Z","iopub.status.idle":"2022-04-23T17:01:13.100784Z","shell.execute_reply.started":"2022-04-23T17:01:13.083893Z","shell.execute_reply":"2022-04-23T17:01:13.098862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.102913Z","iopub.execute_input":"2022-04-23T17:01:13.103253Z","iopub.status.idle":"2022-04-23T17:01:13.116107Z","shell.execute_reply.started":"2022-04-23T17:01:13.1032Z","shell.execute_reply":"2022-04-23T17:01:13.114155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.path = sys.path[:-1]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.119149Z","iopub.execute_input":"2022-04-23T17:01:13.119536Z","iopub.status.idle":"2022-04-23T17:01:13.134831Z","shell.execute_reply.started":"2022-04-23T17:01:13.119495Z","shell.execute_reply":"2022-04-23T17:01:13.133218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sys.path.append(ROOT_DIR/'Mask_RCNN')\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.136891Z","iopub.execute_input":"2022-04-23T17:01:13.1372Z","iopub.status.idle":"2022-04-23T17:01:13.149503Z","shell.execute_reply.started":"2022-04-23T17:01:13.13716Z","shell.execute_reply":"2022-04-23T17:01:13.148356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FashionConfig(Config):\n    \"\"\"Configuration for training on the toy shapes dataset.\n    Derives from the base Config class and overrides values specific\n    to the toy shapes dataset.\n    \"\"\"\n    # Give the configuration a recognizable name\n    NAME = \"class\"\n\n    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    BACKBONE = 'resnet50'\n    # Number of classes (including background)\n    NUM_CLASSES = 1 + len(categories_df)  # background + 46 classes\n\n    # Use small images for faster training. Set the limits of the small side\n    # the large side, and that determines the image shape.\n    IMAGE_MIN_DIM = 256\n    IMAGE_MAX_DIM = 256\n\n    # Use smaller anchors because our image and objects are small\n    RPN_ANCHOR_SCALES = (8, 16, 32, 64, 128)  # anchor side in pixels\n\n    # Reduce training ROIs per image because the images are small and have\n    # few objects. Aim to allow ROI sampling to pick 33% positive ROIs.\n    TRAIN_ROIS_PER_IMAGE = 32\n\n    # Use a small epoch since the data is simple\n    STEPS_PER_EPOCH = 100\n\n    # use small validation steps since the epoch is small\n    VALIDATION_STEPS = 5\n    \nconfig = FashionConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.151577Z","iopub.execute_input":"2022-04-23T17:01:13.152413Z","iopub.status.idle":"2022-04-23T17:01:13.182474Z","shell.execute_reply.started":"2022-04-23T17:01:13.152358Z","shell.execute_reply":"2022-04-23T17:01:13.18017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FashionDataset(utils.Dataset):\n    def __init__(self, df):\n        super().__init__(self)\n        \n        self.IMAGE_SIZE = 256\n        \n        # Add classes\n        for cat in label_desc['categories']:\n            self.add_class('fashion', cat.get('id'), cat.get('name'))\n        \n        # Add images\n        for i, row in df.iterrows():\n            self.add_image(\"fashion\", \n                           image_id=row.name, \n                           path=str(DATA_DIR/'train'/row.name) + '.jpg', \n                           labels=row['ClassId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n            \n    def _resize_image(self, image_path):\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (self.IMAGE_SIZE, self.IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n        return img\n        \n    def load_image(self, image_id):\n        return self._resize_image(self.image_info[image_id]['path'])\n    \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [x for x in info['labels']]\n    \n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((self.IMAGE_SIZE, self.IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (self.IMAGE_SIZE, self.IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.18446Z","iopub.execute_input":"2022-04-23T17:01:13.184834Z","iopub.status.idle":"2022-04-23T17:01:13.215387Z","shell.execute_reply.started":"2022-04-23T17:01:13.184785Z","shell.execute_reply":"2022-04-23T17:01:13.214057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = FashionDataset(image_df)\ndataset.prepare()\n\nfor i in range(6):\n    image_id = random.choice(dataset.image_ids)\n\n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:13.218093Z","iopub.execute_input":"2022-04-23T17:01:13.218787Z","iopub.status.idle":"2022-04-23T17:01:28.209888Z","shell.execute_reply.started":"2022-04-23T17:01:13.218708Z","shell.execute_reply":"2022-04-23T17:01:28.208726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training on a very small subset of the data for fast results for now.\n# image_df = image_df.iloc[0:2]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:28.211276Z","iopub.execute_input":"2022-04-23T17:01:28.211713Z","iopub.status.idle":"2022-04-23T17:01:28.21567Z","shell.execute_reply.started":"2022-04-23T17:01:28.211665Z","shell.execute_reply":"2022-04-23T17:01:28.214135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This code partially supports k-fold training, \n# you can specify the fold to train and the total number of folds here\nFOLD = 0\nN_FOLDS = 2\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits):\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold()\n\ntrain_dataset = FashionDataset(train_df)\ntrain_dataset.prepare()\n\nvalid_dataset = FashionDataset(valid_df)\nvalid_dataset.prepare()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:28.217424Z","iopub.execute_input":"2022-04-23T17:01:28.2177Z","iopub.status.idle":"2022-04-23T17:01:36.663907Z","shell.execute_reply.started":"2022-04-23T17:01:28.217662Z","shell.execute_reply":"2022-04-23T17:01:36.662386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape)\nprint(valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:36.665229Z","iopub.execute_input":"2022-04-23T17:01:36.665483Z","iopub.status.idle":"2022-04-23T17:01:36.67459Z","shell.execute_reply.started":"2022-04-23T17:01:36.665439Z","shell.execute_reply":"2022-04-23T17:01:36.672373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# Note that any hyperparameters here, such as LR, may still not be optimal\nLR = 1e-4\nEPOCHS = [1, 6, 8]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:36.678147Z","iopub.execute_input":"2022-04-23T17:01:36.678522Z","iopub.status.idle":"2022-04-23T17:01:36.691927Z","shell.execute_reply.started":"2022-04-23T17:01:36.678478Z","shell.execute_reply":"2022-04-23T17:01:36.690209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's load the COCO dataset weights to our Model.","metadata":{}},{"cell_type":"code","source":"model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# Load weights trained on MS COCO, but skip layers that\n# are different due to the different number of classes\n# See README for instructions to download the COCO weights\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True,\n                   exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \n                            \"mrcnn_bbox\", \"mrcnn_mask\"])","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:36.695572Z","iopub.execute_input":"2022-04-23T17:01:36.696135Z","iopub.status.idle":"2022-04-23T17:01:44.600375Z","shell.execute_reply.started":"2022-04-23T17:01:36.696016Z","shell.execute_reply":"2022-04-23T17:01:44.59864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5) # only horizontal flip here\n])","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.605388Z","iopub.execute_input":"2022-04-23T17:01:44.606005Z","iopub.status.idle":"2022-04-23T17:01:44.613347Z","shell.execute_reply.started":"2022-04-23T17:01:44.605963Z","shell.execute_reply":"2022-04-23T17:01:44.610526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nmodel_path = 'iMaterialist2_mask_rcnn.h5'\nmodel.load_weights(model_path, by_name=True)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.615355Z","iopub.execute_input":"2022-04-23T17:01:44.615806Z","iopub.status.idle":"2022-04-23T17:01:44.632612Z","shell.execute_reply.started":"2022-04-23T17:01:44.615717Z","shell.execute_reply":"2022-04-23T17:01:44.631595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\"\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR*2, # train heads with higher lr to speedup learning\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model.keras_model.history.history\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.634616Z","iopub.execute_input":"2022-04-23T17:01:44.634991Z","iopub.status.idle":"2022-04-23T17:01:44.657004Z","shell.execute_reply.started":"2022-04-23T17:01:44.634948Z","shell.execute_reply":"2022-04-23T17:01:44.65603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[1],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.658629Z","iopub.execute_input":"2022-04-23T17:01:44.659148Z","iopub.status.idle":"2022-04-23T17:01:44.674348Z","shell.execute_reply.started":"2022-04-23T17:01:44.659106Z","shell.execute_reply":"2022-04-23T17:01:44.673494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR/5,\n            epochs=EPOCHS[2],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.675963Z","iopub.execute_input":"2022-04-23T17:01:44.676535Z","iopub.status.idle":"2022-04-23T17:01:44.693526Z","shell.execute_reply.started":"2022-04-23T17:01:44.676495Z","shell.execute_reply":"2022-04-23T17:01:44.691905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = 'iMaterialist2_mask_rcnn.h5'\nmodel.keras_model.save_weights(model_path)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:01:44.695797Z","iopub.execute_input":"2022-04-23T17:01:44.696504Z","iopub.status.idle":"2022-04-23T17:02:06.348678Z","shell.execute_reply.started":"2022-04-23T17:01:44.696457Z","shell.execute_reply":"2022-04-23T17:02:06.346959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport itertools\nsub_list = []\nmissing_count = 0\nfor i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n    print(row)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:06.352546Z","iopub.execute_input":"2022-04-23T17:02:06.353023Z","iopub.status.idle":"2022-04-23T17:02:09.224707Z","shell.execute_reply.started":"2022-04-23T17:02:06.352962Z","shell.execute_reply":"2022-04-23T17:02:09.22336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# model.train(train_dataset, valid_dataset,\n#             learning_rate=LR,\n#             epochs=EPOCHS[1],\n#             layers='all',\n#             augmentation=augmentation)\n\n# new_history = model.keras_model.history.history\n# for k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.23339Z","iopub.execute_input":"2022-04-23T17:02:09.233734Z","iopub.status.idle":"2022-04-23T17:02:09.25288Z","shell.execute_reply.started":"2022-04-23T17:02:09.233694Z","shell.execute_reply":"2022-04-23T17:02:09.241262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# model.train(train_dataset, valid_dataset,\n#             learning_rate=LR/5,\n#             epochs=EPOCHS[2],\n#             layers='all',\n#             augmentation=augmentation)\n\n# new_history = model.keras_model.history.history\n# for k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.254683Z","iopub.execute_input":"2022-04-23T17:02:09.25509Z","iopub.status.idle":"2022-04-23T17:02:09.271066Z","shell.execute_reply.started":"2022-04-23T17:02:09.25504Z","shell.execute_reply":"2022-04-23T17:02:09.26693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(EPOCHS[0])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.277371Z","iopub.execute_input":"2022-04-23T17:02:09.277652Z","iopub.status.idle":"2022-04-23T17:02:09.600825Z","shell.execute_reply.started":"2022-04-23T17:02:09.277618Z","shell.execute_reply":"2022-04-23T17:02:09.59766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.60186Z","iopub.status.idle":"2022-04-23T17:02:09.60217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.603007Z","iopub.status.idle":"2022-04-23T17:02:09.603294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'/kaggle/working/class*/mask_rcnn_class_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:02:09.604182Z","iopub.status.idle":"2022-04-23T17:02:09.604486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = sample_sub_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:44.138383Z","iopub.execute_input":"2022-04-23T17:04:44.138972Z","iopub.status.idle":"2022-04-23T17:04:44.144453Z","shell.execute_reply.started":"2022-04-23T17:04:44.138918Z","shell.execute_reply":"2022-04-23T17:04:44.143464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(FashionConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)\n\nassert model_path != '', \"Provide path to trained weights\"\nmodel_path = '/kaggle/input/features/fashion20210615T2252/mask_rcnn_fashion_0060.h5'\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:44.159368Z","iopub.execute_input":"2022-04-23T17:04:44.159992Z","iopub.status.idle":"2022-04-23T17:04:54.114704Z","shell.execute_reply.started":"2022-04-23T17:04:44.159948Z","shell.execute_reply":"2022-04-23T17:04:54.113104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '/kaggle/input/imaterialist/iMaterialist2_mask_rcnn.h5'\nmodel_path","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.11734Z","iopub.execute_input":"2022-04-23T17:04:54.11768Z","iopub.status.idle":"2022-04-23T17:04:54.127468Z","shell.execute_reply.started":"2022-04-23T17:04:54.117637Z","shell.execute_reply":"2022-04-23T17:04:54.125799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert data to run-length encoding\ndef to_rle(bits):\n    rle = []\n    pos = 0\n    for bit, group in itertools.groupby(bits):\n        group_list = list(group)\n        if bit:\n            rle.extend([pos, sum(group_list)])\n        pos += len(group_list)\n    return rle","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.128861Z","iopub.execute_input":"2022-04-23T17:04:54.129116Z","iopub.status.idle":"2022-04-23T17:04:54.140412Z","shell.execute_reply.started":"2022-04-23T17:04:54.129083Z","shell.execute_reply":"2022-04-23T17:04:54.139376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Since the submission system does not permit overlapped masks, we have to fix them\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.14202Z","iopub.execute_input":"2022-04-23T17:04:54.142301Z","iopub.status.idle":"2022-04-23T17:04:54.166929Z","shell.execute_reply.started":"2022-04-23T17:04:54.142265Z","shell.execute_reply":"2022-04-23T17:04:54.164436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 256","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.170279Z","iopub.execute_input":"2022-04-23T17:04:54.170738Z","iopub.status.idle":"2022-04-23T17:04:54.183482Z","shell.execute_reply.started":"2022-04-23T17:04:54.170695Z","shell.execute_reply":"2022-04-23T17:04:54.182186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.186312Z","iopub.execute_input":"2022-04-23T17:04:54.186601Z","iopub.status.idle":"2022-04-23T17:04:54.204809Z","shell.execute_reply.started":"2022-04-23T17:04:54.186567Z","shell.execute_reply":"2022-04-23T17:04:54.202045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df=sample_sub_df","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.206797Z","iopub.execute_input":"2022-04-23T17:04:54.207124Z","iopub.status.idle":"2022-04-23T17:04:54.222953Z","shell.execute_reply.started":"2022-04-23T17:04:54.207085Z","shell.execute_reply":"2022-04-23T17:04:54.221433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\"\nimport itertools\nsub_list = []\nmissing_count = 0\nfor i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n    image = resize_image(str(DATA_DIR/'test'/row['ImageId']) + '.jpg')\n    result = model.detect([image])[0]\n    if result['masks'].size > 0:\n        print(result)\n        masks, _ = refine_masks(result['masks'], result['rois'])\n        #print(\"result \",result)\n        for m in range(masks.shape[-1]):\n            mask = masks[:, :, m].ravel(order='F')\n            rle = to_rle(mask)\n            label = result['class_ids'][m] - 1\n            sub_list.append([row['ImageId'], ' '.join(list(map(str, rle))), label, np.NaN])\n    else:\n        # The system does not allow missing ids, this is an easy way to fill them \n        sub_list.append([row['ImageId'], '1 1', 23, np.NaN])\n        missing_count += 1\n        print(\"no\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.224725Z","iopub.execute_input":"2022-04-23T17:04:54.225432Z","iopub.status.idle":"2022-04-23T17:04:54.245414Z","shell.execute_reply.started":"2022-04-23T17:04:54.225383Z","shell.execute_reply":"2022-04-23T17:04:54.243288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.247647Z","iopub.execute_input":"2022-04-23T17:04:54.248034Z","iopub.status.idle":"2022-04-23T17:04:54.267723Z","shell.execute_reply.started":"2022-04-23T17:04:54.247991Z","shell.execute_reply":"2022-04-23T17:04:54.26635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(sub_list, columns=sample_df.columns.values)\nprint(\"Total image results: \", submission_df['ImageId'].nunique())\nprint(\"Missing Images: \", missing_count)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.269142Z","iopub.execute_input":"2022-04-23T17:04:54.269401Z","iopub.status.idle":"2022-04-23T17:04:54.297895Z","shell.execute_reply.started":"2022-04-23T17:04:54.269376Z","shell.execute_reply":"2022-04-23T17:04:54.296174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission_df.to_csv(\"../submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.29961Z","iopub.execute_input":"2022-04-23T17:04:54.30002Z","iopub.status.idle":"2022-04-23T17:04:54.306314Z","shell.execute_reply.started":"2022-04-23T17:04:54.29998Z","shell.execute_reply":"2022-04-23T17:04:54.3043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0):\n        image_id = sample_df.sample()['ImageId'].values[0]\n        image_path = str(DATA_DIR/'test'/image_id)+'.jpg'\n\n        img = cv2.imread(image_path)\n        print(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        result = model.detect([resize_image(image_path)])\n        r = result[0]\n\n        if r['masks'].size > 0:\n            masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n            for m in range(r['masks'].shape[-1]):\n                masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                            (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n\n            y_scale = img.shape[0]/IMAGE_SIZE\n            x_scale = img.shape[1]/IMAGE_SIZE\n            rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n\n            masks, rois = refine_masks(masks, rois)\n        else:\n            masks, rois = r['masks'], r['rois']\n\n        visualize.display_instances(img, rois, masks, r['class_ids'], valid_dataset.class_names, r['scores'],\n                                    title=image_id, figsize=(12, 12))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.308319Z","iopub.execute_input":"2022-04-23T17:04:54.308631Z","iopub.status.idle":"2022-04-23T17:04:54.323718Z","shell.execute_reply.started":"2022-04-23T17:04:54.308593Z","shell.execute_reply":"2022-04-23T17:04:54.322535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport itertools\nsub_list = []\nmissing_count = 0\nfor i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n    image_path =str(DATA_DIR/'test'/row['ImageId']) + '.jpg'\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    if r['masks'].size > 0:\n        print(\"ok\")\n        r = result[0]\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n        visualize.display_instances(img, rois, masks, r['class_ids'], valid_dataset.class_names, r['scores'],title=image_id, figsize=(12, 12))\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    #visualize.display_instances(img, rois, masks, r['class_ids'], valid_dataset.class_names, r['scores'],title=image_id, figsize=(12, 12))","metadata":{"execution":{"iopub.status.busy":"2022-04-23T17:04:54.325976Z","iopub.execute_input":"2022-04-23T17:04:54.326289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(9):\n    image_id=sample_df.sample()['ImageId'].values[0]  \n    image_path = str(DATA_DIR/'test'/image_id)+'.jpg'\n    print(image_path)\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([resize_image(image_path)])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/IMAGE_SIZE\n        x_scale = img.shape[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+label_names, r['scores'],\n                                title=image_id, figsize=(12, 12))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls /kaggle/input/imaterialist1/mask_rcnn_class_0001.h5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(FashionConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=ROOT_DIR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '/kaggle/input/imaterialist1/mask_rcnn_class_0001.h5'\nmodel_path = '/kaggle/input/imaterialist/iMaterialist2_mask_rcnn.h5'\nmodel = modellib.MaskRCNN(mode=\"inference\", model_dir=ROOT_DIR, config=inference_config)\n\n# Load weights trained on MS-COCO\nmodel.load_weights(model_path, by_name=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.sample()['ImageId'].values[0]  \nimage_path = str(DATA_DIR/'test'/image_id)+ '.jpg'\nprint(image_path)\nimg = cv2.imread(image_path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = model.detect([resize_image(image_path)])\nr = result[0]\nr","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}