{"cells":[{"metadata":{},"cell_type":"markdown","source":"> Welcome to the world where fashion meets computer vision! This is a starter kernel that applies Mask R-CNN with COCO pretrained weights to the task of [iMaterialist (Fashion) 2019 at FGVC6](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6)."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport json\nimport glob\nimport random\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\nfrom sklearn.model_selection import StratifiedKFold, KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input')\nROOT_DIR = Path('/kaggle/working')\nIMAGE_DIR = Path('/kaggle/input/imaterialist-fashion-2019-FGVC6/train')\n\n# For demonstration purpose, the classification ignores attributes (only categories),\n# and the image size is set to 512, which is the same as the size of submission masks\n\nNUM_CATS = 46\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dowload Libraries and Pretrained Weights"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# !git clone https://www.github.com/matterport/Mask_RCNN.git\n# os.chdir('Mask_RCNN')\n\n# !rm -rf .git # to prevent an error when the kernel is committed\n# !rm -rf images assets # to prevent displaying images at the bottom of a kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pwd","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# sys.path.append(ROOT_DIR/'Mask_RCNN')\n# from mrcnn.config import Config\n# from mrcnn import utils\n# import mrcnn.model as modellib\n# from mrcnn import visualize\n# from mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n# !ls -lh mask_rcnn_coco.h5\n\n# COCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class FashionConfig(Config):\n#     NAME = \"fashion\"\n#     NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n#     GPU_COUNT = 1\n#     IMAGES_PER_GPU = 4 # a memory error occurs when IMAGES_PER_GPU is too high\n    \n#     BACKBONE = 'resnet50'\n    \n#     IMAGE_MIN_DIM = IMAGE_SIZE\n#     IMAGE_MAX_DIM = IMAGE_SIZE    \n#     IMAGE_RESIZE_MODE = 'none'\n    \n#     RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n#     #DETECTION_NMS_THRESHOLD = 0.0\n    \n#     # STEPS_PER_EPOCH should be the number of instances \n#     # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n#     # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n# #     STEPS_PER_EPOCH = 1000\n# #     VALIDATION_STEPS = 200\n#     STEPS_PER_EPOCH = 10\n#     VALIDATION_STEPS = 2\n    \n# config = FashionConfig()\n# config.display()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(DATA_DIR/\"label_descriptions.json\") as f:\n    label_descriptions = json.load(f)\n\nlabel_names = [x['name'] for x in label_descriptions['categories']]\nattribute_names = [x['name'] for x in label_descriptions['attributes']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(label_names))\nprint(len(attribute_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_names[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attribute_names[:30]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df = pd.read_csv(DATA_DIR/\"train.csv\")\n\nprint('le_segment_df',len(segment_df))\nprint(segment_df.head())\nmultilabel_percent = len(segment_df[segment_df['ClassId'].str.contains('_')])/len(segment_df)*100\nprint(f\"Segments that have attributes: {multilabel_percent:.2f}%\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Segments that contain attributes are only 3.46% of data, and [according to the host](https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6/discussion/90643#523135), 80% of images have no attribute. So, in the first step, we can only deal with categories to reduce the complexity of the task."},{"metadata":{"trusted":true},"cell_type":"code","source":"# segment_df['CategoryId'] = segment_df['Class']\nsegment_df['CategoryId'] = segment_df['ClassId'].str.split('_').str[0]\nsegment_df['AttributeId'] = segment_df['ClassId'].str.split('_').str[1:]\n\nprint(\"Total segments: \", len(segment_df))\n\nprint('max_id:',max(list(map(lambda x:int(x),segment_df['CategoryId'] ))))\nsegment_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"####My Code\nsegment_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_img(IMG_FILE):\n    I = cv2.imread(\"/kaggle/input/train/\" + IMG_FILE, cv2.IMREAD_COLOR)\n    I = cv2.cvtColor(I, cv2.COLOR_BGR2RGB)\n    I = cv2.resize(I, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    plt.imshow(I) \n    \ndef complete_make_mask(data,IMG_FILE):\n    mask_list, cat_list = [], []\n    df = data[data.ImageId == IMG_FILE].reset_index(drop = True)\n    H = df.iloc[0,2]\n    W = df.iloc[0,3]\n    \n    print(\"Correct Category :\", sorted(set((list(df.CategoryId)))))\n    # 1d mask \n    \n    for line in df[['EncodedPixels','CategoryId']].iterrows():\n        # 1d mask \n        mask = np.full(H*W,dtype='int',fill_value = -1)\n        \n        EncodedPixels = line[1][0]\n        Category = line[1][1]\n        \n        pixel_loc = list(map(int,EncodedPixels.split(' ')[0::2]))\n        iter_num =  list(map(int,EncodedPixels.split(' ')[1::2]))\n        for p,i in zip(pixel_loc,iter_num):\n            mask[p:(p+i)] = Category\n        mask = mask.reshape(W,H).T\n#         print(Category, mask.shape)\n        mask_list+=[mask]\n        cat_list+=[Category]\n    \n#     print(\"Output :\",sorted(set(list(mask))))\n#     print('mask:\\n',set(list(mask)))\n#     mask = mask.reshape(W,H).T\n    #rle\n#     return mask\n    return cat_list, mask_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_list = os.listdir('/kaggle/input/train/')\nfor k in img_list[:3]:\n    cat_list1, mask_list1 = complete_make_mask(segment_df, k)\n    plt.figure(figsize=[15,15])\n    plt.subplot(3,5,1)\n    show_img(k)\n    plt.title('Input Image')\n    i=1\n    for mask, cat in zip(mask_list1, cat_list1):\n        mask = cv2.resize(mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n        plt.subplot(3,5,i+1)\n        i+=1\n        plt.imshow(mask,cmap='jet')\n        plt.title(label_names[int(cat)])\n    plt.subplots_adjust(wspace=0.4, hspace=-0.65)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seg_att_df = segment_df[[len(x)>0 for x in segment_df['AttributeId']]].reset_index(drop=['index'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seg_att_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Building apparel image and attribute data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"seg_att_df.iloc[30:50]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df = segment_df.groupby('ImageId')['EncodedPixels', 'CategoryId'].agg(lambda x: list(x))\n# image_df.head()\nsize_df = segment_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n# image_df.head()\n\n# image_df = image_df.iloc[:10]\nprint(\"Total images: \", len(image_df))\nimage_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is the custom function that resizes an image as per pre-trained Mask R-CNN model."},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The crucial part is to create a dataset for this task."},{"metadata":{"trusted":true},"cell_type":"code","source":"# class FashionDataset(utils.Dataset):\n\n#     def __init__(self, df):\n#         super().__init__(self)\n        \n#         # Add classes\n#         for i, name in enumerate(label_names):\n#             self.add_class(\"fashion\", i+1, name)\n        \n#         # Add images \n#         for i, row in df.iterrows():\n#             self.add_image(\"fashion\", \n#                            image_id=row.name, \n#                            path=str(DATA_DIR/'train'/row.name), \n#                            labels=row['CategoryId'],\n#                            annotations=row['EncodedPixels'], \n#                            height=row['Height'], width=row['Width'])\n\n#     def image_reference(self, image_id):\n#         info = self.image_info[image_id]\n#         return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n#     def load_image(self, image_id):\n#         return resize_image(self.image_info[image_id]['path'])\n\n#     def load_mask(self, image_id):\n#         info = self.image_info[image_id]\n                \n#         mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n#         labels = []\n        \n#         for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n#             sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n#             annotation = [int(x) for x in annotation.split(' ')]\n            \n#             for i, start_pixel in enumerate(annotation[::2]):\n#                 sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n#             sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n#             sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n#             mask[:, :, m] = sub_mask\n#             labels.append(int(label)+1)\n            \n#         return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize some random images and their masks."},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset = FashionDataset(image_df)\n# dataset.prepare()\n\n# for i in range(8,10):\n# #     image_id = random.choice(dataset.image_ids)\n#     image_id = dataset.image_ids[i]\n#     print(dataset.image_reference(image_id))\n    \n#     image = dataset.load_image(image_id)\n#     mask, class_ids = dataset.load_mask(image_id)\n#     print('mask_shape:',mask.shape)\n#     print('img_shape:',image.shape)\n# #     print(class_ids\n#     print(dataset.class_names)\n#     print(len(dataset.class_names))\n# #     plt.figure()\n# #     plt.imshow(image)\n# #     visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)\n#     visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, the data are partitioned into train and validation sets."},{"metadata":{"trusted":true},"cell_type":"code","source":"# # This code partially supports k-fold training, \n# # you can specify the fold to train and the total number of folds here\n# FOLD = 0\n# N_FOLDS = 5\n\n# kf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\n# splits = kf.split(image_df) # ideally, this should be multilabel stratification\n\n# def get_fold():    \n#     for i, (train_index, valid_index) in enumerate(splits):\n#         if i == FOLD:\n#             return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \n# train_df, valid_df = get_fold()\n\n# train_dataset = FashionDataset(train_df)\n# train_dataset.prepare()\n\n# valid_dataset = FashionDataset(valid_df)\n# valid_dataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize class distributions of the train and validation data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_segments = np.concatenate(train_df['CategoryId'].values).astype(int)\n# print(\"Total train images: \", len(train_df))\n# print(\"Total train segments: \", len(train_segments))\n\n# plt.figure(figsize=(12, 3))\n# values, counts = np.unique(train_segments, return_counts=True)\n# plt.bar(values, counts)\n# plt.xticks(values, label_names, rotation='vertical')\n# plt.show()\n\n# valid_segments = np.concatenate(valid_df['CategoryId'].values).astype(int)\n# print(\"Total validation images: \", len(valid_df))\n# print(\"Total validation segments: \", len(valid_segments))\n\n# plt.figure(figsize=(12, 3))\n# values, counts = np.unique(valid_segments, return_counts=True)\n# plt.bar(values, counts)\n# plt.xticks(values, label_names, rotation='vertical')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note that any hyperparameters here, such as LR, may still not be optimal\n# LR = 1e-4\n# # EPOCHS = [2, 6, 8]\n# EPOCHS = [1, 2, 3]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This section creates a Mask R-CNN model and specifies augmentations to be used."},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\n# model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n#     'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# augmentation = iaa.Sequential([\n#     iaa.Fliplr(0.5) # only horizontal flip here\n# ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, we train only the heads."},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# model.train(train_dataset, valid_dataset,\n#             learning_rate=LR*2, # train heads with higher lr to speedup learning\n#             epochs=EPOCHS[0],\n#             layers='heads',\n#             augmentation=None)\n\n# history = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, all layers are trained."},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# model.train(train_dataset, valid_dataset,\n#             learning_rate=LR,\n#             epochs=EPOCHS[1],\n#             layers='all',\n#             augmentation=augmentation)\n\n# new_history = model.keras_model.history.history\n# for k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Afterwards, we reduce LR and train again."},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# model.train(train_dataset, valid_dataset,\n#             learning_rate=LR/5,\n#             epochs=EPOCHS[2],\n#             layers='all',\n#             augmentation=augmentation)\n\n# new_history = model.keras_model.history.history\n# for k in new_history: history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's visualize training history and choose the best epoch."},{"metadata":{"trusted":true},"cell_type":"code","source":"# epochs = range(EPOCHS[-1])\n\n# plt.figure(figsize=(18, 6))\n\n# plt.subplot(131)\n# plt.plot(epochs, history['loss'], label=\"train loss\")\n# plt.plot(epochs, history['val_loss'], label=\"valid loss\")\n# plt.legend()\n# plt.subplot(132)\n# plt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\n# plt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\n# plt.legend()\n# plt.subplot(133)\n# plt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\n# plt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\n# plt.legend()\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# best_epoch = np.argmin(history[\"val_loss\"]) + 1\n# print(\"Best epoch: \", best_epoch)\n# print(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict"},{"metadata":{},"cell_type":"markdown","source":"The final step is to use our model to predict test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# glob_list = glob.glob(f'/kaggle/working/fashion*/mask_rcnn_fashion_{best_epoch:04d}.h5')\n# model_path = glob_list[0] if glob_list else ''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This cell defines InferenceConfig and loads the best trained model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# class InferenceConfig(FashionConfig):\n#     GPU_COUNT = 1\n#     IMAGES_PER_GPU = 1\n\n# inference_config = InferenceConfig()\n\n# model = modellib.MaskRCNN(mode='inference', \n#                           config=inference_config,\n#                           model_dir=ROOT_DIR)\n\n# assert model_path != '', \"Provide path to trained weights\"\n# print(\"Loading weights from \", model_path)\n# model.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, load the submission data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample_df = pd.read_csv(DATA_DIR/\"sample_submission.csv\")\n# sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is the main prediction steps, along with some helper functions."},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample_df['EncodedPixels'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Convert data to run-length encoding\n# def to_rle(bits):\n#     rle = []\n#     pos = 0\n#     for bit, group in itertools.groupby(bits):\n#         group_list = list(group)\n#         if bit:\n#             rle.extend([pos, sum(group_list)])\n#         pos += len(group_list)\n#     return rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Since the submission system does not permit overlapped masks, we have to fix them\n# def refine_masks(masks, rois):\n#     areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n#     mask_index = np.argsort(areas)\n#     union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n#     for m in mask_index:\n#         masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n#         union_mask = np.logical_or(masks[:, :, m], union_mask)\n#     for m in range(masks.shape[-1]):\n#         mask_pos = np.where(masks[:, :, m]==True)\n#         if np.any(mask_pos):\n#             y1, x1 = np.min(mask_pos, axis=1)\n#             y2, x2 = np.max(mask_pos, axis=1)\n#             rois[m, :] = [y1, x1, y2, x2]\n#     return masks, rois","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# sub_list = []\n# missing_count = 0\n# for i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n#     image = resize_image(str(DATA_DIR/'test'/row['ImageId']))\n#     result = model.detect([image])[0]\n#     if result['masks'].size > 0:\n#         masks, _ = refine_masks(result['masks'], result['rois'])\n#         for m in range(masks.shape[-1]):\n#             mask = masks[:, :, m].ravel(order='F')\n#             rle = to_rle(mask)\n#             label = result['class_ids'][m] - 1\n#             sub_list.append([row['ImageId'], ' '.join(list(map(str, rle))), label])\n#     else:\n#         # The system does not allow missing ids, this is an easy way to fill them \n#         sub_list.append([row['ImageId'], '1 1', 23])\n#         missing_count += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The submission file is created, when all predictions are ready."},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_df = pd.DataFrame(sub_list, columns=sample_df.columns.values)\n# print(\"Total image results: \", submission_df['ImageId'].nunique())\n# print(\"Missing Images: \", missing_count)\n# submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, it's pleasing to visualize the results! Sample images contain both fashion models and predictions from the Mask R-CNN model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(9):\n#     image_id = sample_df.sample()['ImageId'].values[0]\n#     image_path = str(DATA_DIR/'test'/image_id)\n    \n#     img = cv2.imread(image_path)\n#     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n#     result = model.detect([resize_image(image_path)])\n#     r = result[0]\n    \n#     if r['masks'].size > 0:\n#         masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n#         for m in range(r['masks'].shape[-1]):\n#             masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n#                                         (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n#         y_scale = img.shape[0]/IMAGE_SIZE\n#         x_scale = img.shape[1]/IMAGE_SIZE\n#         rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n#         masks, rois = refine_masks(masks, rois)\n#     else:\n#         masks, rois = r['masks'], r['rois']\n        \n#     visualize.display_instances(img, rois, masks, r['class_ids'], \n#                                 ['bg']+label_names, r['scores'],\n#                                 title=image_id, figsize=(12, 12))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****Building Apparel attribute detection model****"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_mask(data):\n    print(data)\n    df = data.reset_index(drop = True)\n    H = df.iloc[0,2]\n    W = df.iloc[0,3]\n    \n#     print(\"Correct Category :\", int(df.CategoryId))\n    # 1d mask \n    mask = np.full(H*W,dtype='int',fill_value = -1)\n    \n    for line in df[['EncodedPixels','CategoryId']].iterrows():\n        # 1d mask \n#         mask = np.full(H*W,dtype='int',fill_value = -1)\n        \n        EncodedPixels = line[1][0]\n        Category = line[1][1]\n        \n        pixel_loc = list(map(int,EncodedPixels.split(' ')[0::2]))\n        iter_num =  list(map(int,EncodedPixels.split(' ')[1::2]))\n        for p,i in zip(pixel_loc,iter_num):\n            mask[p:(p+i)] = Category\n        mask = mask.reshape(W,H).T\n        print(mask)\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE2 = 299","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***Saving the apparel images with attributes in different dataframes and image resizing as per InceptionV3 model***"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_id_list, apparel_img_list, cat_list, att_list = [], [], [],[]\napparel_id_list, att_id_list = [], []\n# for i in range(seg_att_df.shape[0]):\nfor i in range(100):\n#     if i%100==0:\n    print(i)\n    img_id_list+=[seg_att_df['ImageId'][i]]\n    mask1 = make_mask(seg_att_df.iloc[i:i+1])\n    mask1 = cv2.resize(mask1, (IMAGE_SIZE2, IMAGE_SIZE2), interpolation=cv2.INTER_NEAREST)  \n    apparel_img_list+=[mask1]\n    apparel_id_list+=[int(seg_att_df['CategoryId'][i])]\n    cat_list+=[label_names[int(seg_att_df['CategoryId'][i])]]\n    att_id_list+=[seg_att_df['AttributeId'][i]]\n    att_list+=[[attribute_names[int(x)] for x in seg_att_df['AttributeId'][i]]]\nimage_att = pd.DataFrame({'ImageId':img_id_list,'ApparelImage':apparel_img_list,'ApparelId': apparel_id_list, \n                          'ApparelClass':cat_list,'AttributeId':att_id_list,'AttributeType':att_list})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_att.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Plotting apparel class and stating respect. attributes"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(len(image_att)):\nfor i in range(4):\n    plt.figure(figsize=[10,10])\n    plt.imshow(image_att['ApparelImage'][i])\n    plt.title(image_att['ApparelClass'][i]+'\\n'+'; '.join(image_att['AttributeType'][i]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Prepare Image feature vectors for Apparel class images"},{"metadata":{},"cell_type":"markdown","source":"Using ImageNet pre-trained InceptionV3 model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.inception_v3 import InceptionV3,preprocess_input\n# from keras.applications import InceptionV3\nfrom keras.layers import Dense,BatchNormalization,Dropout,Embedding,RepeatVector\nfrom keras.preprocessing.image import load_img, img_to_array\nfrom keras.models import Sequential\nfrom keras.models import Model\nfrom pickle import dump, load\nfrom keras.models import load_model\nimport numpy as np\ninception = InceptionV3(weights='imagenet')\n\n# pop the last softmax layer and freezing the remaining layers (re-structure the model)\ninception.layers.pop()\n#\nfor layer in inception.layers:\n    layer.trainable = False\n\n# building the final model\npre_trained_incept_v3 = Model(input = inception.input,output = inception.layers[-1].output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre_trained_incept_v3.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"train-val data"},{"metadata":{"trusted":true},"cell_type":"code","source":"msk = np.random.rand(len(image_att)) <= 0.8\ntrain_att = image_att[msk].reset_index(drop=True)\nval_att = image_att[~msk].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_att.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_att.shape)\nprint(val_att.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications.inception_v3 import InceptionV3,preprocess_input\nfrom keras.layers import Dense,BatchNormalization,Dropout,Embedding,RepeatVector\nfrom keras.preprocessing.image import load_img, img_to_array\nimport numpy as np\n\nTARGET_SIZE = (299,299) # needed to convert the image as per pre-trained inceptionv3 requirements\n\nimg_feat_list = []\nfor i in range(len(train_att)):\n    img = image_att['ApparelImage'][i]\n    img = np.stack((img,)*3, axis=-1) # creating gray scale to 3-channel image\n    # Converting image to array\n    img_array = img_to_array(img)\n    nimage = preprocess_input(img_array)\n    # Adding one more dimesion\n    nimage = np.expand_dims(nimage, axis=0)    \n    fea_vec = pre_trained_incept_v3.predict(nimage)\n    fea_vec = np.reshape(fea_vec, fea_vec.shape[1]) # reshape from (1, 2048) to (2048, )\n    img_feat_list+=[fea_vec]\ntrain_att['img_feat'] = img_feat_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_att.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_att.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_att['AttributeType'][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Preparing attribute (text) data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\n# as we'll be building it as image captioning model, we need to add some fixed start and end attributes\"\ntrain_att['AttributeId'] = [[92]+x+[93] for x in train_att['AttributeId']]\ntrain_att['AttributeType'] = [['att_start']+x+['att_end'] for x in train_att['AttributeType']]\n\ntotal_train_att = np.concatenate(train_att['AttributeId'].values).astype(int)\nprint(\"Total Apparel images: \", len(train_att))\nprint(\"All atributes throughout apparel images: \", len(total_train_att))\n\nattribute_names+=['att_start','att_end']\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(total_train_att, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, [attribute_names[x] for x in values], rotation='vertical')\nplt.show()\n\n#currently, dropping apparel attributes with freq. < 10\nfinal_val = values[counts>=2]\nfinal_att = [attribute_names[x] for x in final_val]\n\ntrain_att['Final_att'] = [[x for x in z if x in final_att] for z in train_att['AttributeType']]\n\nmax_length = max([len(x) for x in train_att['Final_att']])#changed from max_no -> max_length\nprint('Max. number of attributes:', max_length)#changed from max_no -> max_length\nvocab_size = len(final_att) + 1\nprint('Feature vocab size:', vocab_size)\n\nixtoword = {}\nwordtoix = {}\n\nix = 1\nfor w in final_att:\n    wordtoix[w] = ix\n    ixtoword[ix] = w\n    ix += 1\n# token = Tokenizer(num_words=vocab_size)\n# token.fit_on_texts(final_att)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ixtoword = {}\nwordtoix = {}\nix = 1\nfor w in final_att:\n    wordtoix[w] = ix\n    ixtoword[ix] = w\n    ix += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wordtoix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_att.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Defininf encoder-decoder model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model,Input\nfrom keras.applications.inception_v3 import InceptionV3,preprocess_input\nfrom keras.layers import Embedding,Dense,BatchNormalization,Dropout,LSTM,add\nfrom keras.utils import plot_model\n\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.np_utils import to_categorical\nimport numpy as np\n\ndef combined_model(MAX_LENGTH,VOCAB_SIZE):\n    \"model parameters\"\n#    NPIX = 299 # required image shape for pre-trained inceptionnv3 model \n#    TARGET_SIZE = (NPIX,NPIX,3)\n    EMBEDDING_SIZE = 256 #\n    \n    # partial caption sequence model    \n    inputs2 = Input(shape=(MAX_LENGTH,))\n    se1 = Embedding(VOCAB_SIZE, EMBEDDING_SIZE, mask_zero=True)(inputs2)\n    se2 = Dropout(0.5)(se1)\n    se3 = LSTM(EMBEDDING_SIZE)(se2) \n    \n    \n    # image feature extractor model\n    inputs1 = Input(shape=(2048,)) # iceptionnv3\n    fe1 = Dropout(0.5)(inputs1)\n    fe2 = Dense(EMBEDDING_SIZE, activation='relu')(fe1)\n    \n    \n    \n    decoder1 = add([fe2, se3])\n    decoder2 = Dense(EMBEDDING_SIZE, activation='relu')(decoder1) \n    #decoder2 = Dense(50, activation='relu')(decoder1) \n    outputs = Dense(VOCAB_SIZE, activation='softmax')(decoder2)\n    \n    \n    # merge the two input models\n    # image_feature + partial caption ===> output\n    model = Model(inputs=[inputs1, inputs2], outputs=outputs) \n    \n    # setting wight of embedded matrix that we saved earlier for words\n#     with open(\"embedding_matrix.pkl\",\"rb\") as f:\n#         embedding_matrix = load(f)   \n#     model.layers[2].set_weights([embedding_matrix])\n#     model.layers[2].trainable = False\n    model.compile(loss='categorical_crossentropy', optimizer='adam')\n\n    return model\n\ndef data_generator(train_att, MAX_LENGTH,VOCAB_SIZE, num_photos_per_batch):\n    X1, X2, y = list(), list(), list()\n    n=0\n    for i in range(len(train_att)):\n        n+=1\n        photo = train_att['img_feat'][i]\n        att_list = list(train_att['Final_att'][i])\n        \n        seq = [wordtoix[x] for x in att_list]\n        for i in range(1,len(seq)):\n            in_seq , op_seq = seq[:i],seq[i]\n            #converting input sequence to fix length\n            in_seq = pad_sequences([in_seq],maxlen=MAX_LENGTH,padding=\"post\")[0]\n            # converting op_seq to vocabulary size\n#                    print(op_seq)\n            op_seq = to_categorical([op_seq],num_classes=VOCAB_SIZE)[0]\n#                    try:\n#                        op_seq = to_categorical([op_seq],num_classes=VOCAB_SIZE)[0]\n#                    except:\n#                        op_seq = np.array([0]*VOCAB_SIZE)\n            X1.append(photo)\n            X2.append(in_seq)\n            y.append(op_seq)\n        # yield the batch data\n        if n==num_photos_per_batch:\n            yield [[np.array(X1), np.array(X2)], np.array(y)]\n            X1, X2, y = list(), list(), list()\n            n=0\n\n                \n# image feature extracted file\ntrain_image_extracted = train_att['img_feat']\n\n#\"load train attributes\ntrain_descriptions = train_att['Final_att']\n\n\nmodel = combined_model(max_length, vocab_size) #\n\nepochs = 30\n\n\nlen(train_descriptions)\n\n\nfor i in range(epochs):\n    batch_size = number_pics_per_batch = 5\n    steps = len(train_descriptions)//number_pics_per_batch\n    generator = data_generator(train_att,max_length, vocab_size,number_pics_per_batch)\n    model.fit_generator(generator, epochs=1, steps_per_epoch=steps, verbose=1)\n\natt_prediction_model = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"att_prediction_model = model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Prediction on new apparel"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# extract features from each photo in the directory\ndef extract_features(img):\n#     print(img)\n    img = np.stack((img,)*3, axis=-1) # creating gray scale to 3-channel image\n#     print(img)\n    # Converting image to array\n    img_array = img_to_array(img)\n    nimage = preprocess_input(img_array)\n    # Adding one more dimesion\n    nimage = np.expand_dims(nimage, axis=0)    \n    fea_vec = pre_trained_incept_v3.predict(nimage)\n    fea_vec = np.reshape(fea_vec, fea_vec.shape[1]) # reshape from (1, 2048) to (2048, )\n    return fea_vec\n\n\n# generate a description for an image\ndef generate_desc(model, photo, max_length):\n    # seed the generation process\n    sequence = ['att_start']\n    photo = photo.reshape(1,2048)\n\n    # iterate over the whole length of the sequence\n    for i in range(max_length):\n        # integer encode input sequence\n        seq = [wordtoix[x] for x in sequence]\n        # pad input\n        seq1 = pad_sequences([seq], maxlen=max_length)\n        # predict next word\n        yhat = model.predict([photo,seq1], verbose=0)\n        # convert probability to integer\n        yhat = np.argmax(yhat)\n        # map integer to word\n        word = ixtoword[yhat]\n        # stop if we cannot map the word\n        if word is None:\n            break\n        # append as input for generating the next word\n        sequence+=[word]\n        # stop if we predict the end of the sequence\n        if word == 'att_end':\n            break\n    return sequence\n        \n \n\n\"prediction on new images\"\nval_att = train_att.copy()\n\n\n\nfor i in range(3):\n    print(i)\n    img = val_att['ApparelImage'][i]\n    plt.figure()\n    plt.imshow(img)\n    photo = extract_features(img)\n    description = generate_desc(att_prediction_model, photo, max_length)\n    plt.title('pred. apparel attributes:\\n'+'; '.join([x for x in description if x not in ['att_start', 'att_end']]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_att.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    print(i)\n    mask_img = val_att['ApparelImage'][i]\n    plt.figure()\n    image_name = \"/kaggle/input/train/\"+val_att['ImageId'][i]\n#     print(image_name)\n    original_image = plt.imread(image_name)\n    plt.imshow(original_image)\n    photo = extract_features(mask_img)\n    description = generate_desc(att_prediction_model, photo, max_length)\n    plt.title('pred. apparel attributes:\\n'+'; '.join([x for x in description if x not in ['att_start', 'att_end']]))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    print(i)\n    img = val_att['ApparelImage'][i]\n    print(val_att[\"ImageId\"][i])\n    image_name = \"/kaggle/input/train/\"+ val_att[\"ImageId\"][i]\n    plt.figure()\n    photo = extract_features(img)\n    description = generate_desc(att_prediction_model, photo, max_length)\n    plt.title('pred. apparel attributes:\\n'+'; '.join([x for x in description if x not in ['att_start', 'att_end']]))\n    image= plt.imread(image_name)\n    plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}