{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":2937133,"sourceType":"datasetVersion","datasetId":1733438},{"sourceId":7428962,"sourceType":"datasetVersion","datasetId":3625455}],"dockerImageVersionId":30132,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing all the necessary libraries","metadata":{}},{"cell_type":"code","source":"# Library to silence Tensorflow Logs\n! pip install -q silence-tensorflow","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:42:47.542458Z","iopub.execute_input":"2023-08-28T12:42:47.542844Z","iopub.status.idle":"2023-08-28T12:43:01.389452Z","shell.execute_reply.started":"2023-08-28T12:42:47.542731Z","shell.execute_reply":"2023-08-28T12:43:01.388520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\n# Add Folders to path, required to import them\nsys.path.append('../input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/efficientnetv2')\nsys.path.append('../input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/Mask_RCNN')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:01.391774Z","iopub.execute_input":"2023-08-28T12:43:01.392055Z","iopub.status.idle":"2023-08-28T12:43:01.399840Z","shell.execute_reply.started":"2023-08-28T12:43:01.392016Z","shell.execute_reply":"2023-08-28T12:43:01.399063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install LZ4 Compression/Decompression Library\n!pip install -q ../input/maskrcnn-tf-2-efficientnetv2-caching/lz4-3.1.3-cp37-cp37m-manylinux1_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:01.400846Z","iopub.execute_input":"2023-08-28T12:43:01.401054Z","iopub.status.idle":"2023-08-28T12:43:09.742910Z","shell.execute_reply.started":"2023-08-28T12:43:01.401030Z","shell.execute_reply":"2023-08-28T12:43:09.741956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import silence_tensorflow.auto\nimport os\nimport json\nimport datetime\nimport numpy as np\nimport pandas as pd\nimport skimage\nimport skimage.draw\nfrom skimage.draw import polygon\nimport time\nimport cv2\nimport matplotlib.pyplot as plt\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log\nfrom os import listdir\nimport tensorflow as tf\nimport random\n# ignore warnings to make outputs clearer\nimport warnings\nimport skimage\nimport imageio\nimport glob\nimport imgaug\nimport multiprocessing\nimport seaborn as sns\nfrom collections import Counter\n\nfrom PIL import Image, ImageDraw\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, train_test_split\nfrom PIL import Image, ImageEnhance\nfrom mrcnn import visualize\nimport sys\nfrom IPython.display import FileLink\nfrom imgaug import augmenters as iaa\n\nwarnings.filterwarnings('ignore')\ntqdm.pandas()\n\nprint(f'Python Version: {sys.version}')\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Tensorflow Keras Version: {tf.keras.__version__}')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:09.746628Z","iopub.execute_input":"2023-08-28T12:43:09.746897Z","iopub.status.idle":"2023-08-28T12:43:16.277860Z","shell.execute_reply.started":"2023-08-28T12:43:09.746864Z","shell.execute_reply":"2023-08-28T12:43:16.276175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:16.279266Z","iopub.execute_input":"2023-08-28T12:43:16.279651Z","iopub.status.idle":"2023-08-28T12:43:16.468459Z","shell.execute_reply.started":"2023-08-28T12:43:16.279609Z","shell.execute_reply":"2023-08-28T12:43:16.467294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Pre-processing","metadata":{}},{"cell_type":"markdown","source":"## Reading input dataframe with some pre-processing","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/deepfashion2-256x256/DeepFashion2 Resized/input/train.csv')\nvalidation_df = pd.read_csv('/kaggle/input/deepfashion2-256x256/DeepFashion2 Resized/input/validation.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:16.470366Z","iopub.execute_input":"2023-08-28T12:43:16.470954Z","iopub.status.idle":"2023-08-28T12:43:30.119409Z","shell.execute_reply.started":"2023-08-28T12:43:16.470849Z","shell.execute_reply":"2023-08-28T12:43:30.118502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Minor pre-processing\ntrain_df['path'] = train_df['path'].str.replace('working', 'input/deepfashion2-256x256/DeepFashion2 Resized')\nvalidation_df['path'] = validation_df['path'].str.replace('working', 'input/deepfashion2-256x256/DeepFashion2 Resized')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:30.120767Z","iopub.execute_input":"2023-08-28T12:43:30.121292Z","iopub.status.idle":"2023-08-28T12:43:30.805116Z","shell.execute_reply.started":"2023-08-28T12:43:30.121245Z","shell.execute_reply":"2023-08-28T12:43:30.804257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_df['segmentation'] = validation_df['segmentation'].progress_apply(lambda x: json.loads(x.replace(\"'\", \"\\\"\")))\ntrain_df['segmentation'] = train_df['segmentation'].progress_apply(lambda x: json.loads(x.replace(\"'\", \"\\\"\")))\n\nvalidation_df['b_box'] = validation_df['b_box'].progress_apply(lambda x: json.loads(x.replace(\"'\", \"\\\"\")))\ntrain_df['b_box'] = train_df['b_box'].progress_apply(lambda x: json.loads(x.replace(\"'\", \"\\\"\")))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:30.810188Z","iopub.execute_input":"2023-08-28T12:43:30.812683Z","iopub.status.idle":"2023-08-28T12:43:57.255958Z","shell.execute_reply.started":"2023-08-28T12:43:30.812632Z","shell.execute_reply":"2023-08-28T12:43:57.255006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df.tail())\nprint(\"\\n\\n\")\ndisplay(validation_df.tail())","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:57.257465Z","iopub.execute_input":"2023-08-28T12:43:57.257861Z","iopub.status.idle":"2023-08-28T12:43:57.324097Z","shell.execute_reply.started":"2023-08-28T12:43:57.257821Z","shell.execute_reply":"2023-08-28T12:43:57.323077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sampling the dataset","metadata":{}},{"cell_type":"code","source":"def balanced_sampling(df, n_samples_per_category):\n    # Store which images (by path) have been sampled\n    sampled_images = set()\n    \n    # Output dataframe to store sampled rows\n    sampled_df = pd.DataFrame()\n\n    # Loop over categories\n    for category in df['category_name'].unique():\n        # Find unique images containing this category\n        category_images = set(df[df['category_name'] == category]['path'].unique())\n        \n        # Exclude already sampled images to ensure new samples\n        available_images = category_images - sampled_images\n        if len(available_images) < n_samples_per_category:\n            selected_images = available_images\n        else:\n            selected_images = set(np.random.choice(list(available_images), size=n_samples_per_category, replace=False))\n        \n        # Add selected images to the sampled_images set\n        sampled_images = sampled_images.union(selected_images)\n\n        # Gather all rows related to the selected images and append to sampled_df\n        sampled_df = pd.concat([sampled_df, df[df['path'].isin(selected_images)]], axis=0)\n\n    return sampled_df.sample(frac=1).reset_index(drop=True)  # shuffle and return","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:57.327584Z","iopub.execute_input":"2023-08-28T12:43:57.327946Z","iopub.status.idle":"2023-08-28T12:43:57.337725Z","shell.execute_reply.started":"2023-08-28T12:43:57.327907Z","shell.execute_reply":"2023-08-28T12:43:57.336905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = balanced_sampling(train_df, 1300) #1320\nvalidation_df = balanced_sampling(validation_df, 1500) #400","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:57.339147Z","iopub.execute_input":"2023-08-28T12:43:57.339566Z","iopub.status.idle":"2023-08-28T12:43:59.947866Z","shell.execute_reply.started":"2023-08-28T12:43:57.339527Z","shell.execute_reply":"2023-08-28T12:43:59.947064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train = train_df.groupby('path')\ngrouped_validation = validation_df.groupby('path')\n\nagg_funcs = {\n    'b_box': list,\n    'category_name': list,\n    'segmentation': list,\n    'category_id': list,\n    'img_height': 'first',\n    'img_width': 'first'\n}\n\ntrain_df = grouped_train.agg(agg_funcs).reset_index()\nvalidation_df = grouped_validation.agg(agg_funcs).reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:43:59.952316Z","iopub.execute_input":"2023-08-28T12:43:59.952553Z","iopub.status.idle":"2023-08-28T12:44:04.762755Z","shell.execute_reply.started":"2023-08-28T12:43:59.952526Z","shell.execute_reply":"2023-08-28T12:44:04.761946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df.tail())\nprint(\"\\n\\n\")\ndisplay(validation_df.tail())\nprint(\"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:04.763987Z","iopub.execute_input":"2023-08-28T12:44:04.764289Z","iopub.status.idle":"2023-08-28T12:44:04.840242Z","shell.execute_reply.started":"2023-08-28T12:44:04.764253Z","shell.execute_reply":"2023-08-28T12:44:04.839483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(15, 45), sharey=True)\n\ncombinations_df_train = pd.DataFrame()\ncombinations_df_validation = pd.DataFrame()\n\n# # Count the occurrences of each category combination\ncombinations_df_train['category_name'] = train_df['category_name']\ncombinations_df_train['category_combination'] = combinations_df_train['category_name'].apply(lambda x: ', '.join(sorted(x)))\ncounter = Counter(combinations_df_train['category_combination'])\ncombinations_df_train = combinations_df_train.join(pd.DataFrame(list(counter.items()), columns=['Category Combination', 'Count']))\nsns.barplot(data=combinations_df_train, y='Category Combination', x='Count', ax=axes[0]).set_title('Category pair distribution in Training Set')\n\ncombinations_df_validation['category_name'] = validation_df['category_name']\ncombinations_df_validation['category_combination'] = combinations_df_validation['category_name'].apply(lambda x: ', '.join(sorted(x)))\ncounter = Counter(combinations_df_validation['category_combination'])\ncombinations_df_validation = combinations_df_validation.join(pd.DataFrame(list(counter.items()), columns=['Category Combination', 'Count']))\nsns.barplot(data=combinations_df_validation, y='Category Combination', x='Count', ax=axes[1]).set_title('Category pair distribution in Validation Set')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:04.841790Z","iopub.execute_input":"2023-08-28T12:44:04.842073Z","iopub.status.idle":"2023-08-28T12:44:10.833827Z","shell.execute_reply.started":"2023-08-28T12:44:04.842036Z","shell.execute_reply":"2023-08-28T12:44:10.832905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_df, test_df = train_test_split(validation_df, test_size=0.50, random_state=42) ","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:10.834834Z","iopub.execute_input":"2023-08-28T12:44:10.835077Z","iopub.status.idle":"2023-08-28T12:44:10.850845Z","shell.execute_reply.started":"2023-08-28T12:44:10.835046Z","shell.execute_reply":"2023-08-28T12:44:10.850101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(22, 10), sharey=True)\n\ncategories = [category for sublist in train_df['category_name'] for category in sublist]\ncounter = Counter(categories)\ncategories = pd.DataFrame(list(counter.items()), columns=['Category', 'Count'])\nsns.barplot(data=categories, x='Category', y='Count', ax=axes[0], order=categories.sort_values('Count',ascending = False).Category).set_title('Category distribution in Training Set')\naxes[0].tick_params('x', labelrotation=90)\n\ncategories = [category for sublist in validation_df['category_name'] for category in sublist]\ncounter = Counter(categories)\ncategories = pd.DataFrame(list(counter.items()), columns=['Category', 'Count'])\nsns.barplot(data=categories, x='Category', y='Count', ax=axes[1], order=categories.sort_values('Count',ascending = False).Category).set_title('Category distribution in Validation Set')\naxes[1].tick_params('x', labelrotation=90)\n\ncategories = [category for sublist in test_df['category_name'] for category in sublist]\ncounter = Counter(categories)\ncategories = pd.DataFrame(list(counter.items()), columns=['Category', 'Count'])\nsns.barplot(data=categories, x='Category', y='Count', ax=axes[2], order=categories.sort_values('Count',ascending = False).Category).set_title('Category distribution in Test Set')\naxes[2].tick_params('x', labelrotation=90)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:10.852256Z","iopub.execute_input":"2023-08-28T12:44:10.853089Z","iopub.status.idle":"2023-08-28T12:44:11.751829Z","shell.execute_reply.started":"2023-08-28T12:44:10.853050Z","shell.execute_reply":"2023-08-28T12:44:11.751067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(validation_df)%2 != 0:\n    random_index = validation_df.sample(n=1).index\n    validation_df = validation_df.drop(random_index)\n    \n    \nif len(train_df)%2 != 0:\n    random_index = train_df.sample(n=1).index\n    train_df = train_df.drop(random_index)\n    \nif len(test_df)%2 != 0:\n    random_index = test_df.sample(n=1).index\n    test_df = test_df.drop(random_index)\n    \n    \n    \nprint(len(validation_df))\nprint(len(train_df))\nprint(len(test_df))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:11.753085Z","iopub.execute_input":"2023-08-28T12:44:11.754367Z","iopub.status.idle":"2023-08-28T12:44:11.776324Z","shell.execute_reply.started":"2023-08-28T12:44:11.754324Z","shell.execute_reply":"2023-08-28T12:44:11.775508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_instances_train = train_df['category_id'].apply(len).tolist()\nnum_instances_validation = validation_df['category_id'].apply(len).tolist()\nnum_instances_test = test_df['category_id'].apply(len).tolist()\n\nfig, axs = plt.subplots(1, 3, figsize=(20, 5), sharey=True, tight_layout=True)\n\nsns.countplot(x=num_instances_train, ax=axs[0]).set_title('Training Dataset')\naxs[0].set_xlabel('Number of Instances')\n\nsns.countplot(x=num_instances_validation, ax=axs[1]).set_title('Validation Dataset')\naxs[1].set_xlabel('Number of Instances')\n\nsns.countplot(x=num_instances_test, ax=axs[2]).set_title('Test Dataset')\naxs[2].set_xlabel('Number of Instances')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:11.777721Z","iopub.execute_input":"2023-08-28T12:44:11.778169Z","iopub.status.idle":"2023-08-28T12:44:12.473769Z","shell.execute_reply.started":"2023-08-28T12:44:11.778129Z","shell.execute_reply":"2023-08-28T12:44:12.473010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def sampling_df(df, n_samples):\n#     proportions = df['Category Combination'].value_counts(normalize=True)\n    \n#     samples = []\n#     for label, proportion in proportions.items():\n#         n_samples_for_label = int(n_samples * proportion)\n#         sampled_df = df[df['Category Combination'] == label].sample(n=n_samples_for_label)\n#         samples.append(sampled_df)\n        \n#     sampled_data = pd.concat(samples, axis=0)\n#     sampled_data = sampled_data.sample(frac=1).reset_index(drop=True)\n#     return sampled_data","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:12.475255Z","iopub.execute_input":"2023-08-28T12:44:12.475729Z","iopub.status.idle":"2023-08-28T12:44:12.482394Z","shell.execute_reply.started":"2023-08-28T12:44:12.475688Z","shell.execute_reply":"2023-08-28T12:44:12.481638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Configuration","metadata":{}},{"cell_type":"code","source":"# Target Image Dimensions which are divisable by 64 as required by the MASK-RCNN model\nHEIGHT_TARGET = 256\nWIDTH_TARGET = 256\nSHAPE_TARGET = (HEIGHT_TARGET, WIDTH_TARGET)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:12.483723Z","iopub.execute_input":"2023-08-28T12:44:12.484365Z","iopub.status.idle":"2023-08-28T12:44:12.490436Z","shell.execute_reply.started":"2023-08-28T12:44:12.484326Z","shell.execute_reply":"2023-08-28T12:44:12.489512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClothDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__()\n        self.df = df\n\n    def load_dataset(self):\n        categories = [\"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n                      \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"]\n        for index, category in enumerate(categories):\n            self.add_class(\"fashion\", index+1, category.lower())\n            \n        for vertical_flip in [True, False]:\n            for horizontal_flip in [True, False]:\n                for index, row in self.df.iterrows():\n                    self.add_image('fashion', \n                                   image_id=index, \n                                   path=row['path'], \n                                   bounding_box=row['b_box'], \n                                   segmentation=row['segmentation'],\n                                   category_name=row['category_name'], \n                                   category_id=row['category_id'],\n                                   img_height=row['img_height'],\n                                   img_width=row['img_width'],\n                                   vertical_flip=vertical_flip, \n                                   horizontal_flip=horizontal_flip)\n            \n    def extract_boxes(self, image_id):\n        image_info = self.image_info[image_id]\n        boxes = np.array(image_info['bounding_box'])  # Convert to numpy array\n        category_names = image_info['category_name']\n        return boxes, category_names, image_info['img_width'], image_info['img_height']\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n    \n        img_width = info['img_width']\n        img_height = info['img_height']\n\n        mask_list = info['segmentation']\n\n        # Initialize an empty mask with all zeros\n        masks = np.zeros([img_height, img_width, len(mask_list)], dtype='uint8')\n\n        # For each mask in the mask_list\n        for i, masks_per_instance in enumerate(mask_list):\n            # Masks for each instance could be multiple in case an instance has disjoint parts; combine them into a single mask here.\n            full_mask = np.zeros([img_height, img_width], dtype='uint8')\n            for seg_mask in masks_per_instance:\n                rr, cc = polygon(seg_mask[1::2], seg_mask[0::2], (img_height, img_width))\n                full_mask[rr, cc] = 1\n            masks[:, :, i] = full_mask\n\n        # Convert the list of category IDs into an array\n        class_ids = np.array(info['category_id'], dtype='int32')\n        return masks, class_ids\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:12.491955Z","iopub.execute_input":"2023-08-28T12:44:12.492281Z","iopub.status.idle":"2023-08-28T12:44:12.510035Z","shell.execute_reply.started":"2023-08-28T12:44:12.492242Z","shell.execute_reply":"2023-08-28T12:44:12.508884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = ClothDataset(train_df)\ndataset.load_dataset()\ndataset.prepare()\n\nplt.figure(figsize=(25, 80))\nfor i in range(5):\n    ax = plt.subplot(1, 5, i+1)\n    image_id = random.choice(dataset.image_ids)\n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    bbox = utils.extract_bboxes(mask)\n    print(dataset.image_reference(image_id))\n    visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names, ax=ax)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:12.511616Z","iopub.execute_input":"2023-08-28T12:44:12.511871Z","iopub.status.idle":"2023-08-28T12:44:21.747685Z","shell.execute_reply.started":"2023-08-28T12:44:12.511838Z","shell.execute_reply":"2023-08-28T12:44:21.746994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET_LENGTH = len(train_df)\nMAX_BATCH = 2\nBATCH_SIZE = sorted([int(DATASET_LENGTH/n) for n in range(1,DATASET_LENGTH+1) if DATASET_LENGTH % n ==0 and DATASET_LENGTH/n<=MAX_BATCH],reverse=True)[0]  \nSTEPS = int(DATASET_LENGTH/BATCH_SIZE)\nSTEPS","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:21.748804Z","iopub.execute_input":"2023-08-28T12:44:21.750368Z","iopub.status.idle":"2023-08-28T12:44:21.763350Z","shell.execute_reply.started":"2023-08-28T12:44:21.750324Z","shell.execute_reply":"2023-08-28T12:44:21.761101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VAL_DATASET_LENGTH = len(validation_df)\nMAX_BATCH = 2\nVAL_BATCH_SIZE = sorted([int(VAL_DATASET_LENGTH/n) for n in range(1,VAL_DATASET_LENGTH+1) if VAL_DATASET_LENGTH % n ==0 and VAL_DATASET_LENGTH/n<=MAX_BATCH],reverse=True)[0]  \nVAL_STEPS = int(VAL_DATASET_LENGTH/VAL_BATCH_SIZE)\nVAL_STEPS","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:21.765045Z","iopub.execute_input":"2023-08-28T12:44:21.765781Z","iopub.status.idle":"2023-08-28T12:44:21.776349Z","shell.execute_reply.started":"2023-08-28T12:44:21.765730Z","shell.execute_reply":"2023-08-28T12:44:21.775498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClothConfig(Config):\n    # name of the configuration\n    NAME = \"fashion_config\"\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 2\n    STEPS_PER_EPOCH = STEPS\n    VALIDATION_STEPS = VAL_STEPS\n\n    \n    # Number of Classes\n    NUM_CLASSES = 1 + 13\n\n    # Image Dimensions\n    IMAGE_MIN_DIM = HEIGHT_TARGET\n    IMAGE_MAX_DIM = WIDTH_TARGET\n    IMAGE_SHAPE = [HEIGHT_TARGET, WIDTH_TARGET, 3]\n    IMAGE_RESIZE_MODE = 'square'\n    BACKBONE = 'efficientnetv2-b0'\n    \n    TRAIN_BN = False\n    \n    # Learning Rate\n    LEARNING_RATE = 0.0001\n    WEIGHT_DECAY = 0.0\n    LR_SCHEDULE = False\n    \n    # Dataloader Queue Size (was set to 100 but resulted in OOM error)\n    MAX_QUEUE_SIZE = 3\n    \n    # Cache Items\n    CACHE = False\n    \n    # Debug mode will disable model checkpoints\n    DEBUG = False\n    \n    # Do not use multithreading as this slows down the dataloader!\n    WORKERS = 0\n    \n    # Losses\n    LOSS_WEIGHTS = {\n        'rpn_class_loss': 1.0,    # is the class of the bbox correct? / RPN anchor classifier loss (Forground/Background)\n        'rpn_bbox_loss': 1.0,     # is the size of the bbox correct? / RPN bounding box loss graph (bbox of generic object)\n        'mrcnn_class_loss': 1.0,  # loss for the classifier head of Mask R-CNN (Background / specific class)\n        'mrcnn_bbox_loss': 1.0,   # is the size of the bounding box correct or not? / loss for Mask R-CNN bounding box refinement\n        'mrcnn_mask_loss': 1.0,   # is the class correct? is the pixel correctly assign to the class? / mask binary cross-entropy loss for the masks head\n    }\n    \n    # Training Structure\n    FPN_CLASSIF_FC_LAYERS_SIZE = 1024\n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    \n    # Regions of Interest\n    PRE_NMS_LIMIT = 1500\n    \n    # Non Max Supression\n    POST_NMS_ROIS_TRAINING = 800\n    POST_NMS_ROIS_INFERENCE = 800\n    \n    # Instances\n    MAX_GT_INSTANCES = 2\n    TRAIN_ROIS_PER_IMAGE = 200\n    DETECTION_MAX_INSTANCES = 3\n    \n    # Thresholds\n    RPN_NMS_THRESHOLD = 0.70        # IoU Threshold for RPN proposals and GT\n    DETECTION_MIN_CONFIDENCE = 0.50 # Non-Background Confidence Threshold\n    DETECTION_NMS_THRESHOLD = 0.30  # IoU Threshold for ROI and GT\n    ROI_POSITIVE_RATIO = 0.33\n    \n    # Prediction Mask Shape\n    MASK_SHAPE = (28, 28)\n    \n    # Size of mask groundtruth\n    USE_MINI_MASK = True\n    MINI_MASK_SHAPE = (56, 56)  # Uncomment this if running into memory issues\n    \n\nconfig = ClothConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:21.778324Z","iopub.execute_input":"2023-08-28T12:44:21.778908Z","iopub.status.idle":"2023-08-28T12:44:21.805558Z","shell.execute_reply.started":"2023-08-28T12:44:21.778870Z","shell.execute_reply":"2023-08-28T12:44:21.804779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"# training set\ntraining_data = ClothDataset(train_df)\ntraining_data.load_dataset()\ntraining_data.prepare()\n\n# validation set\nvalidation_data = ClothDataset(validation_df)\nvalidation_data.load_dataset()\nvalidation_data.prepare()\n\n# load fashion config\nconfig = ClothConfig()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:44:21.808487Z","iopub.execute_input":"2023-08-28T12:44:21.808711Z","iopub.status.idle":"2023-08-28T12:44:33.941227Z","shell.execute_reply.started":"2023-08-28T12:44:21.808684Z","shell.execute_reply":"2023-08-28T12:44:33.940387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CHECKPOINT_DIRECTORY = '/kaggle/working/model_checkpoints_head'\nos.makedirs(CHECKPOINT_DIRECTORY)\nmodel = modellib.MaskRCNN(mode='training', model_dir=CHECKPOINT_DIRECTORY, config=config)\n\n# Loading COCO weights for Transfer learning\nCOCO_MODEL_PATH = '../input/maskrcnn-tf-2-efficientnetv2-caching/mask_rcnn_coco.h5'\nmodel.load_weights(COCO_MODEL_PATH, by_name=True, exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\",  \"mrcnn_bbox\", \"mrcnn_mask\", \n                                                           \"fpn_c5p5\", \"fpn_c4p4\", \"fpn_c3p3\", \"fpn_c2p2\"])\n\n\n# Load EfficientNetV2 Weights Pretrained on Imagenet21K \nmodel.keras_model.layers[1].load_weights('/kaggle/input/maskrcnn-tf-2-efficientnetv2-caching/Instance_Segmentation/efficientnetv2_model_checkpoints/efficientnetv2-b0-imagenet21k.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.show_summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initially training just the heads","metadata":{}},{"cell_type":"code","source":"start_train = time.time()\nmodel.train(train_dataset=training_data, \n            val_dataset=validation_data, \n            learning_rate=0.0002, \n            layers='heads',\n            epochs=3,\n            augmentation=None)\nend_train = time.time()\nminutes = round((end_train - start_train) / 60, 2)\nprint(f'Training took {minutes} minutes')\n\nhistory = model.keras_model.history.history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training all layers","metadata":{}},{"cell_type":"code","source":"start_train = time.time()\nmodel.train(train_dataset=training_data, \n            val_dataset=validation_data, \n            learning_rate=0.0001, \n            layers='all',\n            epochs=9,\n            augmentation=augmentation)\nend_train = time.time()\nminutes = round((end_train - start_train) / 60, 2)\nprint(f'Training all took {minutes} minutes')\n\nnew_history = model.keras_model.history.history\nfor k in new_history: \n    history[k] = history[k] + new_history[k]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training all layers again","metadata":{}},{"cell_type":"code","source":"start_train = time.time()\nmodel.train(train_dataset=training_data, \n            val_dataset=validation_data, \n            learning_rate=0.00002, \n            layers='all',\n            epochs=3,\n            augmentation=augmentation)\nend_train = time.time()\nminutes = round((end_train - start_train) / 60, 2)\nprint(f'Training all took {minutes} minutes')\n\nnew_history = model.keras_model.history.history\nfor k in new_history: \n    history[k] = history[k] + new_history[k]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('/kaggle/working/history_efficientNetv2-S.npy',history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"history = np.load(\"/kaggle/input/efficientnetv2b0-training/history_efficientNetv2-S.npy\", allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:45:50.935010Z","iopub.execute_input":"2023-08-28T12:45:50.935358Z","iopub.status.idle":"2023-08-28T12:45:50.948932Z","shell.execute_reply.started":"2023-08-28T12:45:50.935320Z","shell.execute_reply":"2023-08-28T12:45:50.948134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to plot metrics\ndef plot_history_metric(ax, metric, f_best=np.argmax):\n    values = history[metric]\n    N_EPOCHS = len(values)\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n    x_ticks = np.arange(1, N_EPOCHS+1)\n    \n    ax.plot(x_ticks, values, label='train')\n    argmin = f_best(values)\n    ax.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label='train_best')\n    ax.set_ylabel(metric, fontsize=15, labelpad=10)\n    ax.set_xlabel('epoch', fontsize=15, labelpad=10)\n    ax.tick_params(axis='x', labelsize=12)\n    ax.tick_params(axis='y', labelsize=12)\n    ax.set_xticks(x) # set tick step to 1 and let x axis start at 1\n    ax.legend(prop={'size': 15})","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:45:51.438472Z","iopub.execute_input":"2023-08-28T12:45:51.439075Z","iopub.status.idle":"2023-08-28T12:45:51.450806Z","shell.execute_reply.started":"2023-08-28T12:45:51.439037Z","shell.execute_reply":"2023-08-28T12:45:51.449957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:45:52.115495Z","iopub.execute_input":"2023-08-28T12:45:52.115796Z","iopub.status.idle":"2023-08-28T12:45:52.122505Z","shell.execute_reply.started":"2023-08-28T12:45:52.115763Z","shell.execute_reply":"2023-08-28T12:45:52.121659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 3, figsize=(22,18))\nplot_history_metric(axs[0, 0], 'loss', f_best=np.argmin)\nplot_history_metric(axs[0, 1], 'rpn_class_loss', f_best=np.argmin)\nplot_history_metric(axs[0, 2], 'rpn_bbox_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 0], 'mrcnn_class_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 1], 'mrcnn_bbox_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 2], 'mrcnn_mask_loss', f_best=np.argmin)\n\nplot_history_metric(axs[0, 0], 'val_loss', f_best=np.argmin)\nplot_history_metric(axs[0, 1], 'val_rpn_class_loss', f_best=np.argmin)\nplot_history_metric(axs[0, 2], 'val_rpn_bbox_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 0], 'val_mrcnn_class_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 1], 'val_mrcnn_bbox_loss', f_best=np.argmin)\nplot_history_metric(axs[1, 2], 'val_mrcnn_mask_loss', f_best=np.argmin)\n\naxs[0, 0].legend(['loss', 'val_loss'])\naxs[0, 0].grid()\n\naxs[0, 1].legend(['rpn_class_loss', 'val_rpn_class_loss '])\naxs[0, 1].grid()\n\naxs[0, 2].legend(['rpn_bbox_loss', 'val_rpn_bbox_loss'])\naxs[0, 2].grid()\n\naxs[1, 0].legend(['mrcnn_class_loss', 'val_mrcnn_class_loss'])\naxs[1, 0].grid()\n\naxs[1, 1].legend(['mrcnn_bbox_loss', 'val_mrcnn_bbox_loss'])\naxs[1, 1].grid()\n\naxs[1, 2].legend(['mrcnn_mask_loss', 'val_mrcnn_mask_loss'])\naxs[1, 2].grid()\n\n\nfig.tight_layout()\nfig.show() ","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:45:52.741969Z","iopub.execute_input":"2023-08-28T12:45:52.742832Z","iopub.status.idle":"2023-08-28T12:45:55.005300Z","shell.execute_reply.started":"2023-08-28T12:45:52.742794Z","shell.execute_reply":"2023-08-28T12:45:55.004593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Inference","metadata":{}},{"cell_type":"code","source":"glob_list = glob.glob(f'/kaggle/working/model_checkpoints_head/fashion*/mask_rcnn_fashion_config_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel_path","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:46:12.294798Z","iopub.execute_input":"2023-08-28T12:46:12.295106Z","iopub.status.idle":"2023-08-28T12:46:12.303007Z","shell.execute_reply.started":"2023-08-28T12:46:12.295072Z","shell.execute_reply":"2023-08-28T12:46:12.302158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = \"/kaggle/input/efficientnetv2b0-training/mask_rcnn_fashion_config_0012.h5\"","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:46:12.661801Z","iopub.execute_input":"2023-08-28T12:46:12.662317Z","iopub.status.idle":"2023-08-28T12:46:12.666515Z","shell.execute_reply.started":"2023-08-28T12:46:12.662275Z","shell.execute_reply":"2023-08-28T12:46:12.665548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET_LENGTH = len(test_df)\nMAX_BATCH = 2\nBATCH_SIZE = sorted([int(DATASET_LENGTH/n) for n in range(1,DATASET_LENGTH+1) if DATASET_LENGTH % n ==0 and DATASET_LENGTH/n<=MAX_BATCH],reverse=True)[0]  \nSTEPS = int(DATASET_LENGTH/BATCH_SIZE)\nSTEPS","metadata":{"execution":{"iopub.status.busy":"2023-08-28T21:17:01.054592Z","iopub.execute_input":"2023-08-28T21:17:01.054928Z","iopub.status.idle":"2023-08-28T21:17:01.063797Z","shell.execute_reply.started":"2023-08-28T21:17:01.054894Z","shell.execute_reply":"2023-08-28T21:17:01.062989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(ClothConfig):\n    IMAGES_PER_GPU = 1\n    DETECTION_MAX_INSTANCES = 1\n    DETECTION_MIN_CONFIDENCE = 0.50\n    USE_MINI_MASK = False\n    STEPS_PER_EPOCH = STEPS\n    \n\ninference_config = InferenceConfig()\ninference_config.display()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T21:17:02.015706Z","iopub.execute_input":"2023-08-28T21:17:02.016456Z","iopub.status.idle":"2023-08-28T21:17:02.036502Z","shell.execute_reply.started":"2023-08-28T21:17:02.016422Z","shell.execute_reply":"2023-08-28T21:17:02.035642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode=\"inference\", \n                          config=inference_config, \n                          model_dir='/kaggle/working/')\n\n# Set EfficientNetV2 head untrainable\nif 'efficientnetv2-' in  inference_config.BACKBONE:\n    model.keras_model.layers[1].layers[-1].trainable = False\n    \n    \nassert model_path != '', \"Provide path to trained weights\"\nprint(\"\\nLoading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:46:15.905028Z","iopub.execute_input":"2023-08-28T12:46:15.905658Z","iopub.status.idle":"2023-08-28T12:46:37.963520Z","shell.execute_reply.started":"2023-08-28T12:46:15.905617Z","shell.execute_reply":"2023-08-28T12:46:37.962564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation set\ntesting_data = ClothDataset(test_df)\ntesting_data.load_dataset()\ntesting_data.prepare()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:46:37.965252Z","iopub.execute_input":"2023-08-28T12:46:37.965534Z","iopub.status.idle":"2023-08-28T12:46:42.557960Z","shell.execute_reply.started":"2023-08-28T12:46:37.965495Z","shell.execute_reply":"2023-08-28T12:46:42.557132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualising some Predictions","metadata":{}},{"cell_type":"code","source":"label_names = [\"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n              \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"]\nfig, axs = plt.subplots(figsize=(22, 8))\nfor i in range(10):\n    ax = plt.subplot(2, 5, i+1)\n    image_path = test_df.sample()['path'].values[0]\n    \n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    result = model.detect([img])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = img.shape[0]/256\n        x_scale = img.shape[1]/256\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    visualize.display_instances(img, rois, masks, r['class_ids'], \n                                ['bg']+label_names, r['scores'],\n                                title=image_id, ax=ax)\n\n    \nplt.tight_layout()    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:46:42.559328Z","iopub.execute_input":"2023-08-28T12:46:42.559612Z","iopub.status.idle":"2023-08-28T12:46:52.993881Z","shell.execute_reply.started":"2023-08-28T12:46:42.559576Z","shell.execute_reply":"2023-08-28T12:46:52.993098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicted vs Actual","metadata":{}},{"cell_type":"code","source":"def plot_actual_vs_predicted(dataset, model, config, n_images):\n    image_ids = np.random.choice(dataset.image_ids, n_images)\n    fig, axs = plt.subplots(figsize=(30, 15))\n    for i, image_id in enumerate(image_ids):\n        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n            modellib.load_image_gt(dataset, config, image_id)\n        info = dataset.image_info[image_id]\n        print(\"image ID: {}.{} ({}) {}\".format(info[\"source\"], info[\"id\"], image_id, \n                                               dataset.image_reference(image_id)))\n        print(\"Original image shape: \", modellib.parse_image_meta(image_meta[np.newaxis,...])[\"original_image_shape\"][0])\n\n        # Run object detection\n        molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n        results = model.detect([image], verbose=0)\n#         results = model.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1)\n\n        # Display results\n        r = results[0]\n        log(\"gt_class_id\", gt_class_id)\n        log(\"gt_bbox\", gt_bbox)\n        log(\"gt_mask\", gt_mask)\n\n        # Compute AP over range 0.5 to 0.95 and print it\n        utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,\n                               r['rois'], r['class_ids'], r['scores'], r['masks'],\n                               verbose=1)\n        \n        ax = plt.subplot(2, 5, i+1)\n        visualize.display_differences(\n            image,\n            gt_bbox, gt_class_id, gt_mask,\n            r['rois'], r['class_ids'], r['scores'], r['masks'],\n            dataset.class_names,\n            show_box=False, show_mask=False, ax = ax,\n            iou_threshold=0.5, score_threshold=0.5)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:47:01.103684Z","iopub.execute_input":"2023-08-28T12:47:01.103991Z","iopub.status.idle":"2023-08-28T12:47:01.115901Z","shell.execute_reply.started":"2023-08-28T12:47:01.103956Z","shell.execute_reply":"2023-08-28T12:47:01.114988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_actual_vs_predicted(testing_data, model, inference_config, 10)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:47:01.516088Z","iopub.execute_input":"2023-08-28T12:47:01.516902Z","iopub.status.idle":"2023-08-28T12:47:04.714409Z","shell.execute_reply.started":"2023-08-28T12:47:01.516858Z","shell.execute_reply":"2023-08-28T12:47:04.713555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation mAP","metadata":{}},{"cell_type":"code","source":"all_scores = []\nall_class_ids = []\nactual_labels = []\npredicted_labels = []\ndef calculate_mAP(data_loader, threshold=0.5):\n    APs = []\n\n    for image_id in tqdm(data_loader.image_ids):\n        # Load image and ground truth data\n        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n            modellib.load_image_gt(data_loader, inference_config,\n                                   image_id)\n        molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n\n\n        results = model.detect([image], verbose=0)\n        r = results[0]\n\n        # Calculate mAP\n        AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, \n                                                             r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'],\n                                                             iou_threshold=threshold)\n        \n        APs.append(AP)\n        all_scores.extend(r[\"scores\"])\n        all_class_ids.extend(r[\"class_ids\"])\n        actual_labels.extend(gt_class_id)\n        predicted_labels.extend(r[\"class_ids\"])\n        \n\n    print(\"mAP@IoU{:.2f}: {:.2f}\".format(threshold, np.mean(APs)))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T19:51:38.905585Z","iopub.execute_input":"2023-08-28T19:51:38.905880Z","iopub.status.idle":"2023-08-28T19:51:38.916511Z","shell.execute_reply.started":"2023-08-28T19:51:38.905848Z","shell.execute_reply":"2023-08-28T19:51:38.915510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculate_mAP(testing_data)\ncalculate_mAP(testing_data, 0.65)\ncalculate_mAP(testing_data, 0.75)\ncalculate_mAP(testing_data, 0.85)\ncalculate_mAP(testing_data, 0.95)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:47:11.366977Z","iopub.execute_input":"2023-08-28T12:47:11.367830Z","iopub.status.idle":"2023-08-28T18:38:02.374448Z","shell.execute_reply.started":"2023-08-28T12:47:11.367788Z","shell.execute_reply":"2023-08-28T18:38:02.373603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculate_mAP(testing_data)\nprint(all_scores)\nprint(all_class_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Histogram of confidence intervals\nfig, axs = plt.subplots(figsize=(15, 8))\nplt.hist(all_scores, bins=50, facecolor='blue', alpha=0.7)\nplt.title('Histogram of Confidence Scores Across All Garment Categories')\nplt.xlabel('Score')\nplt.ylabel('Number of Predictions')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T21:02:53.019463Z","iopub.execute_input":"2023-08-28T21:02:53.019770Z","iopub.status.idle":"2023-08-28T21:02:53.627058Z","shell.execute_reply.started":"2023-08-28T21:02:53.019736Z","shell.execute_reply":"2023-08-28T21:02:53.626318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxprops = dict(linestyle='-', linewidth=4, color='b')\nmedianprops = dict(linestyle='-', linewidth=4, color='r')\n\ndf = pd.DataFrame({\n    'Scores': all_scores,\n    'Class': all_class_ids\n})\n\ndf.boxplot(column='Scores', by='Class', showmeans=True,\n           whiskerprops=dict(linestyle='-', linewidth=1.5), \n           capprops=dict(linestyle='-', linewidth=1.5),\n           boxprops=boxprops, medianprops=medianprops, figsize=(20, 12))\nplt.title('Box Plot of Confidence Scores by Class Across All Images')\nplt.suptitle('')  # Suppress the default title\nplt.xticks(range(1, 14), labels=[\"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n                 \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"], rotation=45)\nplt.xlabel('Class ID')\nplt.ylabel('Score')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T21:06:43.084716Z","iopub.execute_input":"2023-08-28T21:06:43.084992Z","iopub.status.idle":"2023-08-28T21:06:43.672893Z","shell.execute_reply.started":"2023-08-28T21:06:43.084962Z","shell.execute_reply":"2023-08-28T21:06:43.672152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## mAP for different Garment Categories","metadata":{}},{"cell_type":"markdown","source":"### Confusion matrix for images with only one garment","metadata":{}},{"cell_type":"code","source":"all_scores = []\nall_class_ids = []\nactual_labels = []\npredicted_labels = []\ndef calculate_mAP_conf(data_loader, threshold=0.5):\n    APs = []\n\n    for image_id in tqdm(data_loader.image_ids):\n        # Load image and ground truth data\n        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n            modellib.load_image_gt(data_loader, inference_config,\n                                   image_id)\n        molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n\n\n        results = model.detect([image], verbose=0)\n        r = results[0]\n\n        # Calculate mAP\n        AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, \n                                                             r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'],\n                                                             iou_threshold=threshold)\n        \n        APs.append(AP)\n        if len(r[\"class_ids\"]) == 1 and r[\"class_ids\"][0] != 0:\n            all_scores.append(r[\"scores\"][0])\n            all_class_ids.append(r[\"class_ids\"][0])\n            actual_labels.extend(gt_class_id)\n            predicted_labels.append(r[\"class_ids\"][0])\n\n    print(\"mAP@IoU{:.2f}: {:.2f}\".format(threshold, np.mean(APs)))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T22:06:21.224078Z","iopub.execute_input":"2023-08-28T22:06:21.224695Z","iopub.status.idle":"2023-08-28T22:06:21.237101Z","shell.execute_reply.started":"2023-08-28T22:06:21.224651Z","shell.execute_reply":"2023-08-28T22:06:21.236271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_single_garment_rows(df):\n    \"\"\"Filter rows of a DataFrame that have only one garment.\"\"\"\n    return df[df['category_name'].apply(lambda x: len(x) == 1)]","metadata":{"execution":{"iopub.status.busy":"2023-08-28T22:06:22.063526Z","iopub.execute_input":"2023-08-28T22:06:22.063827Z","iopub.status.idle":"2023-08-28T22:06:22.068607Z","shell.execute_reply.started":"2023-08-28T22:06:22.063795Z","shell.execute_reply":"2023-08-28T22:06:22.067859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df = filter_single_garment_rows(test_df)\nfiltered_data = ClothDataset(filtered_df)\nfiltered_data.load_dataset()\nfiltered_data.prepare()\n\ncalculate_mAP_conf(filtered_data)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T22:06:22.979238Z","iopub.execute_input":"2023-08-28T22:06:22.980036Z","iopub.status.idle":"2023-08-28T22:30:18.410242Z","shell.execute_reply.started":"2023-08-28T22:06:22.979995Z","shell.execute_reply":"2023-08-28T22:30:18.409236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confidence_matrix(actual_labels, predicted_labels, all_scores, class_names):\n    n_classes = len(class_names)\n    \n    # Initialize confusion matrix with zeros\n    confidence_matrix = np.zeros((n_classes, n_classes))\n    count_matrix = np.zeros((n_classes, n_classes))\n    \n    # Populate the matrix\n    for i in range(len(predicted_labels)):\n        confidence_matrix[actual_labels[i], predicted_labels[i]] += all_scores[i]\n        count_matrix[actual_labels[i], predicted_labels[i]] += 1\n    \n    # Normalize the matrix\n    confidence_matrix = np.divide(confidence_matrix, count_matrix, out=np.zeros_like(confidence_matrix), where=count_matrix!=0)\n    \n    mask = np.ones_like(confidence_matrix, dtype=bool)\n    mask[confidence_matrix.argmax(axis=0), np.arange(n_classes)] = False\n    \n    # Plot the matrix\n    plt.figure(figsize=(20, 12))\n    ax = sns.heatmap(confidence_matrix, annot=True, cmap=\"Reds\", cbar=False, mask=mask,\n                     xticklabels=class_names, yticklabels=class_names)\n    sns.heatmap(confidence_matrix, annot=True, cmap=\"Blues\", cbar=False, mask=~mask, vmin=0.52,\n                xticklabels=class_names, yticklabels=class_names, ax=ax)\n    plt.xlabel('Actual labels')\n    plt.ylabel('Predicted labels')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T23:05:36.165646Z","iopub.execute_input":"2023-08-28T23:05:36.166615Z","iopub.status.idle":"2023-08-28T23:05:36.177117Z","shell.execute_reply.started":"2023-08-28T23:05:36.166573Z","shell.execute_reply":"2023-08-28T23:05:36.176366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confidence_matrix(actual_labels, predicted_labels, all_scores, [\"background\", \"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n                 \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T23:05:36.698025Z","iopub.execute_input":"2023-08-28T23:05:36.698280Z","iopub.status.idle":"2023-08-28T23:05:37.730092Z","shell.execute_reply.started":"2023-08-28T23:05:36.698252Z","shell.execute_reply":"2023-08-28T23:05:37.729353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Images with one category only","metadata":{}},{"cell_type":"code","source":"def filter_by_category_name(df, category):\n    \"\"\"Filter rows of a DataFrame where 'category_name' column contains only the specified category.\"\"\"\n    return df[df['category_name'].apply(lambda x: x == [category])]","metadata":{"execution":{"iopub.status.busy":"2023-08-28T23:06:01.882445Z","iopub.execute_input":"2023-08-28T23:06:01.882729Z","iopub.status.idle":"2023-08-28T23:06:01.891515Z","shell.execute_reply.started":"2023-08-28T23:06:01.882700Z","shell.execute_reply":"2023-08-28T23:06:01.890591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for category in [\"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n                 \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"]:\n    filtered_df = filter_by_category_name(test_df, category)\n    filtered_data = ClothDataset(filtered_df)\n    filtered_data.load_dataset()\n    filtered_data.prepare()\n    print(\"\\n\\nEvaluation for images with only \" + category)\n    calculate_mAP(filtered_data)\n    calculate_mAP(filtered_data, 0.60)\n    calculate_mAP(filtered_data, 0.75)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T23:06:03.291798Z","iopub.execute_input":"2023-08-28T23:06:03.292306Z","iopub.status.idle":"2023-08-29T00:14:35.184023Z","shell.execute_reply.started":"2023-08-28T23:06:03.292265Z","shell.execute_reply":"2023-08-29T00:14:35.183269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Images with atleast one category","metadata":{}},{"cell_type":"code","source":"def filter_by_category_name(df, category):\n    \"\"\"Filter rows of a DataFrame where 'category_name' column contains the specified category.\"\"\"\n    return df[df['category_name'].apply(lambda x: category in x)]","metadata":{"execution":{"iopub.status.busy":"2023-08-29T00:25:12.304991Z","iopub.execute_input":"2023-08-29T00:25:12.305306Z","iopub.status.idle":"2023-08-29T00:25:12.310298Z","shell.execute_reply.started":"2023-08-29T00:25:12.305273Z","shell.execute_reply":"2023-08-29T00:25:12.309565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for category in [\"short sleeve top\", \"long sleeve top\", \"short sleeve outwear\", \"long sleeve outwear\", \"vest\", \"sling\", \n                 \"shorts\", \"trousers\", \"skirt\", \"short sleeve dress\", \"long sleeve dress\", \"vest dress\", \"sling dress\"]:\n    filtered_df = filter_by_category_name(test_df, category)\n    filtered_data = ClothDataset(filtered_df)\n    filtered_data.load_dataset()\n    filtered_data.prepare()\n    print(\"\\n\\nEvaluation for images with atleast one \" + category)\n    calculate_mAP(filtered_data)\n    calculate_mAP(filtered_data, 0.60)\n    calculate_mAP(filtered_data, 0.75)","metadata":{"execution":{"iopub.status.busy":"2023-08-29T00:25:15.059819Z","iopub.execute_input":"2023-08-29T00:25:15.060249Z"},"trusted":true},"execution_count":null,"outputs":[]}]}