{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport os\n\n\n#FOLLOWING FUNCTION os.walk() GOES TO EACH DIRECTORY AND RETURNS A 3-TUPLE (DIRPATH,dirs in this dir, Files in this dir)\n\n\n# for dirname, _, filenames in os.walk('/kaggle/input'): # '_' what does this mean\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ls=next(os.walk('/kaggle/input'))\n# ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# type(df['EncodedPixels'][2]) #encoded pixels are stored as string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc  #garbage collector\nimport sys \nimport json\nimport glob\nimport random\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa  #data augmentatoin (multiple representations of the same image')'\nfrom sklearn.model_selection import StratifiedKFold, KFold\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datadir = Path('/kaggle/input/imaterialist-fashion-2020-fgvc7/')\nmodeldir = Path('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_CATS = 3\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')    \n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/working/Mask_RCNN')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.append('/kaggle/working/Mask_RCNN')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sys.path is the list of directories and files pthon goes through looking for modules and files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install tensorflow==1.15.0rc0 #need that because Mask-RCNN version\n# !pip install keras==2.1.5\n\nimport tensorflow\nprint(tensorflow.__version__)\nimport keras\nprint(keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#os.chdir() changed the working directory to Mask_RCNN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mrcnn.config import Config #need to have tensorflow 2.2.0 installed\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5  #wget command in linux is used to download files from the internet\n# !ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionConfig(Config):\n    NAME = \"fashion\"\n    NUM_CLASSES = 3 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1 # a memory error occurs when IMAGES_PER_GPU is too high\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE   #config class uses these variables image_min_dims and image_max_dims, need to feed them the image size\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    STEPS_PER_EPOCH = 100\n    VALIDATION_STEPS = 20\n    \nconfig = FashionConfig()\n# config.display()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(datadir/'label_descriptions.json') as f:\n    label_descriptions=json.load(f)\n    \nlabel_names=[]\nfor x in label_descriptions['categories']:#we'll only take the data for classes 0 and 10\n    if((x['id']==0)&(x['id']==10)):\n        label_names.append(x['name'])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1=df[(df['ClassId']==0)|(df['ClassId']==10)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df=df1.groupby('ImageId')['EncodedPixels','ClassId'].agg(lambda x:list(x))\nsize_df=df1.groupby('ImageId')['Height','Width'].mean()\nimage_df=image_df.join(size_df,on='ImageId')\nimage_df.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Function to resize images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage import io\n\ndef resize_image(image_path):\n    try:\n        img = io.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_AREA)  \n        return img\n    except Exception as e:\n        print(str(e))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(datadir/'label_descriptions.json') as f:\n    label_descriptions=json.load(f)\n    \nlabel_names=[]\nfor x in label_descriptions['categories']:#we'll only take the data for classes 0 and 10\n    if((x['id']==0)|(x['id']==10)):\n        label_names.append(x['name'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionDataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", str(i), name)\n\n                \n        # Add images \n        for i, row in df.iterrows():\n            if(row['ClassId'][0]==10):\n                row['ClassId'][0]=1\n            self.add_image(\"fashion\", \n                           image_id=row.name, \n                           path=str(datadir/'train'/row['ImageId'])+'.jpg',\n                           labels=row['ClassId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path']\n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]  #info refers to info of an image(height,width,labels,annotation..)\n                \n        mask = np.zeros((512, 512,1)), dtype=np.uint8)  #each mask will be the same shape as the image i.e. (512,512,1)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8) #creating a mask of the size of the original image\n            annotation = [int(x) for x in annotation.split(' ')]  #splitting and converting all entries to int\n            \n            \n            #decoding the encoded mask\n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n            \n            #converting it to a 2-D array, resizing it to the shape as that of the input\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (512, 512), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask  #mth mask for that particular image\n            labels.append(int(label))\n            \n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = FashionDataset(image_df)\ndataset.prepare()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset.class_names   BG refers to background","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# len(label_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    image_id = random.choice(dataset.image_ids)\n    print(dataset.image_reference(image_id))\n    \n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    print(class_ids)\n    visualize.display_top_masks(image, mask,class_ids,dataset.class_names, limit=2) #limit of what?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLD = 0\nN_FOLDS = 5\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits): #train_index and valid_index are arrays. iloc will return a df when multiple rows are passed/ returns a series object otherwise  \n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold()\n\ntrain_dataset = FashionDataset(train_df)\ntrain_dataset.prepare()\n\nvalid_dataset = FashionDataset(valid_df)\nvalid_dataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid_df.shape 20% of the data is validation data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_segments = np.concatenate(train_df['ClassId'].values).astype(int)  #concatinating to a single list\nprint(\"Total train images: \", len(train_df))\nprint(\"Total train segments: \", len(train_segments)) #this len can be greater than no of train images since one image could have more than one class id associated\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(train_segments, return_counts=True) #number of zeroes and ones\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\nvalid_segments = np.concatenate(valid_df['ClassId'].values).astype(int)\nprint(\"Total train images: \", len(valid_df))\nprint(\"Total validation segments: \", len(valid_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(valid_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 1e-4\nEPOCHS = [2, 6, 8]\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = modellib.MaskRCNN(mode='training', config=config, model_dir=modeldir)\n\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5) # only horizontal flip here\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR*2, # train heads with higher lr to speedup learning\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=augmentation)\n\nhistory = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df=pd.read_csv(DATA_DIR/'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InferenceConfig(FashionConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=modeldir)\nmodel_path='/kaggle/working/fashion20200518T1903/mask_rcnn_fashion_0002.h5'\n# assert model_path != '/kaggle/working/fashion20200517T2123/mask_rcnn_fashion_001.h5', \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_rle(bits):\n    rle = []\n    pos = 0\n    for bit, group in itertools.groupby(bits):\n        group_list = list(group)\n        if bit:\n            rle.extend([pos, sum(group_list)])\n        pos += len(group_list)\n    return rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# %%time\n# sub_list = []\n# missing_count = 0\n# for i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n#     image = resize_image(str(DATA_DIR/'test'/row['ImageId'])+'.jpg')\n#     result = model.detect([image])[0]\n#     if result['masks'].size > 0:\n#         masks, _ = refine_masks(result['masks'], result['rois'])\n#         for m in range(masks.shape[-1]):\n#             mask = masks[:, :, m].ravel(order='F')\n#             rle = to_rle(mask)\n#             label = result['class_ids'][m] - 1\n#             sub_list.append([row['ImageId'], ' '.join(list(map(str, rle))), label])\n#     else:\n#         # The system does not allow missing ids, this is an easy way to fill them \n#         sub_list.append([row['ImageId'], '1 1', 23])\n#         missing_count += 1\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(4):\n# image_id = sample_df.sample()['ImageId'].values[0]\nimage_id='0002f5a0ebc162ecfb73e2c91e3b8f62'\nimage_path = str(datadir/'train'/image_id)+'.jpg'\n\nimg = cv2.imread(image_path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\nresult = model.detect([resize_image(image_path)])\nr = result[0]\n\nif r['masks'].size > 0:\n    masks = np.zeros((512, 512, r['masks'].shape[-1]), dtype=np.uint8)\n    for m in range(r['masks'].shape[-1]):\n        masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                    (512, 512), interpolation=cv2.INTER_NEAREST)\n\n    y_scale = 512/IMAGE_SIZE\n    x_scale = 512/IMAGE_SIZE\n    rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n\n    masks, rois = refine_masks(masks, rois)\nelse:\n    masks, rois = r['masks'], r['rois']\n# mask2=np.zeros((512, 512))\n# for j in range(masks.shape[2]):\n#     mask2=np.maximum(masks[:,:,j],mask2)\n\n#     masks2=np.maximum(masks[:,:,j],masks2)\n# print(masks2)\n# plt.imshow(masks2)\n# visualize.display_instances(img, rois, masks, r['class_ids'], \n#                             ['bg']+label_names, r['scores'],\n#                             title=image_id, figsize=(12, 12))\nvisualize.display_instances(resize_image(image_path), r['rois'], r['masks'], r['class_ids'], \n                            ['BG','Shirt','Dress'], r['scores'])\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(mask2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}