{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!git config --global http.proxy http://proxyuser:proxypwd@proxy.server.com:8080","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Import required libraries\nimport gc\nimport sys\nimport json\nimport random\nfrom pathlib import Path\nfrom PIL import Image\n#import time\n\nimport cv2 # for image manipulation\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom imgaug import augmenters as iaa\n\nimport seaborn as sns\nimport matplotlib.image as mpimg\nfrom matplotlib import pyplot as plt\n\nimport pickle\nfrom tqdm import tqdm, tqdm_pandas\n#from scipy.signal import argrelextrema\n\nimport itertools\nimport h5py #read .h5\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install urllib3==1.7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install tensorflow==1.5 #need that because Mask-RCNN version\n!pip install keras==2.1.5\n\nimport tensorflow\nprint(tensorflow.__version__)\nimport keras\nprint(keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Take a look at the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image = Image.open(\"/kaggle/input/imaterialist-fashion-2020-fgvc7/train/8617b2102bb25fbb0a93fb7f11e6397c.jpg\")\nimage = np.array(image)\nprint(image.shape)\nplt.show()\nplt.imshow(image)\nplt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataDir = \"/kaggle/input/imaterialist-fashion-2020-fgvc7/\"\nworkDir = \"/kaggle/working/\"\nos.listdir(dataDir)\n\n#alternatively\n#!ls /kaggle/input/imaterialist-fashion-2020-fgvc7/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(os.path.join(dataDir, 'sample_submission.csv'))\nsample_submission.sample(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(os.path.join(dataDir, 'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train data shape: {0} \\nUnique number of train images: {1}'\n      .format(train_data.shape, train_data[\"ImageId\"].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Test data shape: {0} \\nUnique number of test images: {1}'\n      .format(sample_submission.shape, sample_submission[\"ImageId\"].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(train_data['Height'].describe()).T.drop(columns = ['std','25%', '50%', '75%'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(train_data['Width'].describe()).T.drop(columns = ['std','25%', '50%', '75%'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (100,10))\nmax_height = list(set(train_data[train_data['Height'] == train_data['Height'].max()]['ImageId']))[0]\nimage = mpimg.imread('{0}/train/{1}.jpg'.format(dataDir, max_height))                     \nplt.imshow(image)\nplt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"#Extract information from the .json file\nwith open(os.path.join(dataDir, 'label_descriptions.json'), 'r') as file:\n    label_description = json.load(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"label_description","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Classes and Attributes processing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_classes = len(label_description['categories'])\nn_attributes = len(label_description['attributes'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Classes: {0} \\nAttributes: {1}'.\n     format(str(n_classes), str(n_attributes)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories_data = pd.DataFrame(label_description['categories'])\nattributes_data = pd.DataFrame(label_description['attributes'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attributes_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories_data.supercategory.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attributes_data.supercategory.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check out training images and their masks","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images(size = 4, figsize = (12, 12)):\n    #get the images\n    image_ids = train_data['ImageId'].unique()[:size]\n    images = []\n    \n    for image_id in image_ids:\n        images.append(mpimg.imread('{0}/train/{1}.jpg'.format(dataDir, image_id)))\n        \n    count = 0\n    \n    fig, ax = plt.subplots(nrows = 2, ncols = 2, figsize = figsize)\n    for row in ax:\n        for col in row:\n            col.imshow(images[count])\n            col.axis('off')\n            count += 1\n    plt.show()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_images()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Function to create mask\ndef create_mask(size):\n    image_ids = train_data['ImageId'].unique()[:size] #get a number of images\n    images_meta = [] #to be added in this array\n    \n    for image_id in image_ids:\n        img = mpimg.imread('{0}/train/{1}.jpg'.format(dataDir, image_id))\n        images_meta.append({\n            'image': img,\n            'shape': img.shape,\n            'encoded_pixels': train_data[train_data['ImageId'] == image_id]['EncodedPixels'],\n            'class_ids': train_data[train_data['ImageId'] == image_id]['ClassId']\n        })\n        \n    masks = []\n    \n    for image in images_meta:\n        shape = image.get('shape') #get via key\n        encoded_pixels = list(image.get('encoded_pixels')) \n        class_ids = list(image.get('class_ids'))\n        \n        #Initialize numpy array with shape same as image size\n        height, width = shape[:2] \n        mask = np.zeros((height, width)).reshape(-1) \n        # (-1) 'The new shape should be compatible with the original shape'\n        # numpy allow us to give one of new shape parameter as -1 but not (-1, -1)).\n        # It means that it is an unknown dimension and we want numpy to figure it out.\n        # And numpy will figure this by looking at the 'length of the array and remaining\n        # dimensions' and making sure it satisfies the above mentioned criteria\n        \n        #Iterate over encoded pixels and create mask\n        for segment, (pixel_str, class_id) in enumerate(zip(encoded_pixels, class_ids)):\n            splitted_pixels = list(map(int, pixel_str.split()))      #split the pixels string\n            pixel_starts = splitted_pixels[::2]                      #choose every second element\n            run_lengths = splitted_pixels[1::2]                      #start from 1 with step size 2\n            assert max(pixel_starts) < mask.shape[0]                 #make sure it is ok\n            \n            for pixel_start, run_length in zip(pixel_starts, run_lengths):\n                pixel_start = int(pixel_start) - 1\n                run_length = int(run_length)\n                mask[pixel_start:pixel_start+run_length] = 255 - class_id \n        masks.append(mask.reshape((height, width), order = 'F'))\n    \n    return masks, images_meta\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_segmented_images(size = 4, figsize = (14, 14)):\n    #First, create masks from given segments\n    masks, images_meta = create_mask(size)\n    \n    #Plot images\n    \n    count = 0\n    \n    fig, ax = plt.subplots(nrows = 2, ncols = 2, figsize = figsize)\n    for row in ax:\n        for col in row:\n            col.imshow(images_meta[count]['image'])\n            col.imshow(masks[count], alpha = 0.50)\n            col.axis('off')\n            count += 1\n    plt.show()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_segmented_images()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Group data by unique images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"images_data = train_data.groupby('ImageId')['EncodedPixels', 'ClassId'].agg(lambda x: list(x))\ndimensions_data = train_data.groupby('ImageId')['Height', 'Width'].mean()\nimages_data = images_data.join(dimensions_data, on='ImageId')\n\nimages_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total images: \", len(images_data))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Apply Mask-RCNN following instructions [here](https://github.com/matterport/Mask_RCNN/blob/master/samples/shapes/train_shapes.ipynb) and [this GitHub](https://github.com/NovatecConsulting/SemanticSegmentation-Examples/tree/master/RSNA%20Pneumonia%20Detection%20(TF%20with%20Mask%20R-CNN)).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('Mask_RCNN')\n\n!rm -rf .git # to prevent an error when the kernel is committed\n!rm -rf images assets # to prevent displaying images at the bottom of a kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sys.path.append(workDir/'Mask_RCNN') # To find local version of the library\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Directory to save logs and trained model\nmodelDir = os.path.join(workDir, \"logs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionImagesConfig(Config):\n    \"\"\"Configuration for training on the iMaterialist dataset.\n    Derives from the base Config class and overrides values specific\n    to the dataset.\n    \"\"\"\n    # Give the configuration a recognizable name\n    NAME = \"fashion2020\"\n\n    # Train on 1 GPU and 3 images per GPU. We can put multiple images on each\n    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 3\n\n    # Number of classes (including background)\n    NUM_CLASSES = 1 + len(categories_data)  # background + 46 shapes\n    BACKBONE = 'resnet50'\n\n    # Use small images for faster training. Set the limits of the small side\n    # the large side, and that determines the image shape.\n    IMAGE_MIN_DIM = 256\n    IMAGE_MAX_DIM = 256\n\n    # Use smaller anchors because our image and objects are small\n    RPN_ANCHOR_SCALES = (8, 16, 32, 64, 128)  # anchor side in pixels\n\n    # Reduce training ROIs per image because the images are small and have\n    # few objects. \n    TRAIN_ROIS_PER_IMAGE = 16\n\n    # Use a small epoch since the data is simple\n    STEPS_PER_EPOCH = 100\n\n    # use small validation steps since the epoch is small\n    VALIDATION_STEPS = 5\n    \nconfig = FashionImagesConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fashion2020Dataset(utils.Dataset):\n    def __init__(self, data):\n        super().__init__(self)\n        \n        self.IMAGE_SIZE = 256\n        self.DIMENSIONS = (256, 256)\n        \n        for category in label_description['categories']:\n            self.add_class('fashion2020', category.get('id'), category.get('name'))\n            \n        for i, row in data.iterrows():\n            self.add_image('fashion2020',\n                          image_id = row.name,\n                          path = str('{0}/train/{1}.jpg'.format(dataDir, row.name)),\n                          labels = row['ClassId'],\n                          annotations = row['EncodedPixels'],\n                          height = row['Height'],\n                          width = row['Width'])\n            \n    def modify_image(self, image_path):\n        #dims = (self.IMAGE_SIZE, self.IMAGE_SIZE)\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, self.DIMENSIONS, interpolation = cv2.INTER_AREA)\n        \n        return img\n    \n    def load_image(self, image_id):\n        img = self.image_info[image_id]['path']\n        return self.modify_image(img)\n    \n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [x for x in info['labels']]\n    \n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n        mask = np.zeros((self.IMAGE_SIZE, self.IMAGE_SIZE, len(info['annotations'])), dtype = np.uint8)\n        \n        labels = []\n        \n        for (m, (annotation, label)) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel + annotation[2*i+1]] = 1\n                \n            sub_mask = sub_mask.reshape((info['height'], info['width']), order = 'F')\n            sub_mask = cv2.resize(sub_mask, self.DIMENSIONS, interpolation = cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = Fashion2020Dataset(images_data)\ndataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Show masks separately\nfor i in range(10):\n    image_id = random.choice(dataset.image_ids)\n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit = 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load random image and mask.\nimage_id = random.choice(dataset.image_ids)\nimage = dataset.load_image(image_id)\nmask, class_ids = dataset.load_mask(image_id)\n# Compute Bounding box\nbbox = utils.extract_bboxes(mask)\n\n# Display image and additional stats\nprint(\"image_id \", image_id, dataset.image_reference(image_id))\nlog(\"image\", image)\nlog(\"mask\", mask)\nlog(\"class_ids\", class_ids)\nlog(\"bbox\", bbox)\n# Display image and instances\nvisualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Split to training and validation data\nfrom sklearn.utils import shuffle\n\nrandom.seed(42)\nimages_data_shuffled = shuffle(images_data)\nval_size = int(0.05 * len(images_data_shuffled['ClassId']))\nimage_data_val = images_data_shuffled[:val_size]\nimage_data_train = images_data_shuffled[val_size:]\n\nprint(len(image_data_train), len(image_data_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Training set: {image_data_train.shape} \\nValidation set: {image_data_val.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare the training dataset\ndataset_train = Fashion2020Dataset(image_data_train)\ndataset_train.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_val = Fashion2020Dataset(image_data_val)\ndataset_val.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_ids = [0]\nwhile class_ids[0] == 0:  ## look for a mask\n    image_id = random.choice(dataset_train.image_ids)\n    image_fp = dataset_train.image_reference(image_id)\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n\nprint(image.shape)\n\nplt.figure(figsize=(12, 12))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\nfor i in range(mask.shape[2]):\n    masked += image[:, :, 0] * mask[:, :, i]\nplt.imshow(masked, cmap = 'gray', alpha = 0.75)\nplt.axis('off')\n\nprint(image_fp)\nprint(class_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Apply some image augmentation, incl. flipping, rotation, blurring, etc.\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## geometric transform\n        iaa.Affine(\n            scale={\"x\": (0.98, 1.02), \"y\": (0.98, 1.04)},\n            translate_percent={\"x\": (-0.02, 0.02), \"y\": (-0.06, 0.06)},\n            rotate=(-3, 3),\n        ),\n        iaa.Fliplr(0.2)\n    ]),\n    iaa.OneOf([ ## brightness or contrast or blur\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.7, 1.1)),\n        iaa.GaussianBlur(sigma=(0.0, 0.2)),\n    ]),\n])\n\n# test augmentation on image\nimggrid = augmentation.draw_grid(image[:, :, 0], cols=5, rows=2)\nplt.figure(figsize=(30, 12))\nplt.axis('off')\n_ = plt.imshow(imggrid[:, :, 0], cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = modellib.MaskRCNN(mode = 'training', config = config, model_dir = workDir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    \"mrcnn_class_logits\", \"mrcnn_bbox_fc\",\n    \"mrcnn_bbox\", \"mrcnn_mask\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#copy this class, only to uncomment the tensorboard callback in the train function\nclass MaskRCNN():\n    def train(self, train_dataset, val_dataset, learning_rate, epochs, layers,\n              augmentation=None, custom_callbacks=None, no_augmentation_sources=None):\n        \"\"\"Train the model.\n        train_dataset, val_dataset: Training and validation Dataset objects.\n        learning_rate: The learning rate to train with\n        epochs: Number of training epochs. Note that previous training epochs\n                are considered to be done alreay, so this actually determines\n                the epochs to train in total rather than in this particaular\n                call.\n        layers: Allows selecting wich layers to train. It can be:\n            - A regular expression to match layer names to train\n            - One of these predefined values:\n              heads: The RPN, classifier and mask heads of the network\n              all: All the layers\n              3+: Train Resnet stage 3 and up\n              4+: Train Resnet stage 4 and up\n              5+: Train Resnet stage 5 and up\n        augmentation: Optional. An imgaug (https://github.com/aleju/imgaug)\n            augmentation. For example, passing imgaug.augmenters.Fliplr(0.5)\n            flips images right/left 50% of the time. You can pass complex\n            augmentations as well. This augmentation applies 50% of the\n            time, and when it does it flips images right/left half the time\n            and adds a Gaussian blur with a random sigma in range 0 to 5.\n\n                augmentation = imgaug.augmenters.Sometimes(0.5, [\n                    imgaug.augmenters.Fliplr(0.5),\n                    imgaug.augmenters.GaussianBlur(sigma=(0.0, 5.0))\n                ])\n\t    custom_callbacks: Optional. Add custom callbacks to be called\n\t        with the keras fit_generator method. Must be list of type keras.callbacks.\n        no_augmentation_sources: Optional. List of sources to exclude for\n            augmentation. A source is string that identifies a dataset and is\n            defined in the Dataset class.\n        \"\"\"\n        assert self.mode == \"training\", \"Create model in training mode.\"\n\n        # Pre-defined layer regular expressions\n        layer_regex = {\n            # all layers but the backbone\n            \"heads\": r\"(mrcnn\\_.*)|(rpn\\_.*)|(fpn\\_.*)\",\n            # From a specific Resnet stage and up\n            \"3+\": r\"(res3.*)|(bn3.*)|(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\\_.*)|(rpn\\_.*)|(fpn\\_.*)\",\n            \"4+\": r\"(res4.*)|(bn4.*)|(res5.*)|(bn5.*)|(mrcnn\\_.*)|(rpn\\_.*)|(fpn\\_.*)\",\n            \"5+\": r\"(res5.*)|(bn5.*)|(mrcnn\\_.*)|(rpn\\_.*)|(fpn\\_.*)\",\n            # All layers\n            \"all\": \".*\",\n        }\n        if layers in layer_regex.keys():\n            layers = layer_regex[layers]\n\n        # Data generators\n        train_generator = data_generator(train_dataset, self.config, shuffle=True,\n                                         augmentation=augmentation,\n                                         batch_size=self.config.BATCH_SIZE,\n                                         no_augmentation_sources=no_augmentation_sources)\n        val_generator = data_generator(val_dataset, self.config, shuffle=True,\n                                       batch_size=self.config.BATCH_SIZE)\n\n        # Create log_dir if it does not exist\n        if not os.path.exists(self.log_dir):\n            os.makedirs(self.log_dir)\n\n        # Callbacks\n        callbacks = [\n            keras.callbacks.TensorBoard(log_dir=self.log_dir,\n                                        histogram_freq=0, write_graph=True, write_images=False),\n            keras.callbacks.ModelCheckpoint(self.checkpoint_path,\n                                            verbose=1, save_weights_only=True),\n        ]\n\n        # Add custom callbacks to the list\n        if custom_callbacks:\n            callbacks += custom_callbacks\n\n        # Train\n        log(\"\\nStarting at epoch {}. LR={}\\n\".format(self.epoch, learning_rate))\n        log(\"Checkpoint Path: {}\".format(self.checkpoint_path))\n        self.set_trainable(layers)\n        self.compile(learning_rate, self.config.LEARNING_MOMENTUM)\n\n        # Work-around for Windows: Keras fails on Windows when using\n        # multiprocessing workers. See discussion here:\n        # https://github.com/matterport/Mask_RCNN/issues/13#issuecomment-353124009\n        if os.name is 'nt':\n            workers = 0\n        else:\n            workers = multiprocessing.cpu_count()\n        print(workers)\n        self.keras_model.fit_generator(\n            train_generator,\n            initial_epoch=self.epoch,\n            epochs=epochs,\n            steps_per_epoch=self.config.STEPS_PER_EPOCH,\n            callbacks=callbacks,\n            validation_data=val_generator,\n            validation_steps=self.config.VALIDATION_STEPS,\n            max_queue_size=100,\n            workers=1,\n            use_multiprocessing=False,\n        )\n        self.epoch = max(self.epoch, epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# These may not be optimal parameters just to run it!\nLEARNING_RATE = 0.005\nLEARNING_RATE_TUNE = 0.0001 #for the last 3 epochs\nEPOCHS = [1, 3, 5, 8] \n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train heads with higher lr for more learning speed\n\"\"\"model.train(dataset_train, dataset_val,\n            learning_rate = LEARNING_RATE,\n            epochs = EPOCHS[0],\n            layers = 'heads',\n            augmentation = None)  \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#history = model.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"%cd ..\nmodel.keras_model.save_weights('modelHead.h5')\npickle.dump(history, open('modelHead.pkl', 'ab'))\n%cd $workDir\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_weights('/kaggle/input/head-saved-weights/modelHead.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#now with all layers and augmentation included 2 more epochs\n\"\"\"model.train(dataset_train, dataset_val,\n            learning_rate = LEARNING_RATE,\n            epochs = EPOCHS[1],\n            layers = 'all',\n            augmentation = augmentation)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load history\n#history = pickle.load(open('/kaggle/input/head-saved-weights/modelHead.pkl', 'rb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"new_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"%cd ..\nmodel.keras_model.save_weights('modelAll1.h5')\npickle.dump(history, open('modelAll1.pkl', 'ab'))\n%cd $workDir\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#decrease learning rate and train for 2 more epochs\n\"\"\"model.train(dataset_train, dataset_val,\n            learning_rate = LEARNING_RATE/5,\n            epochs = EPOCHS[2],\n            layers = 'all',\n            augmentation = augmentation)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"new_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"%cd ..\nmodel.keras_model.save_weights('modelAll2.h5')\npickle.dump(history, open('modelAll2.pkl', 'ab'))\n%cd $workDir\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#the last three epochs train with LR = 1e-4\n\"\"\"model.train(dataset_train, dataset_val,\n            learning_rate = LEARNING_RATE_TUNE,\n            epochs = EPOCHS[3],\n            layers = 'all',\n            augmentation = augmentation)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"new_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"model.keras_model.save_weights('modelAll3.h5')\npickle.dump(history, open('modelAll3.pkl', 'wb'))\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = pickle.load(open('/kaggle/input/head-saved-weights/modelAll3.pkl', 'rb'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> ### Plot training results ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = range(1, len(next(iter(history.values())))+1) #get number of epochs\nhistory_data = pd.DataFrame(history, index=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"\"\"\"%cd ..\nhistory_data.to_csv('History data from Mask_RCNN training' + '.csv')\n%cd $workDir\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(40,8))\n\nplt.subplot(141)\nplt.plot(epochs, history_data[\"loss\"], label=\"Train loss\")\nplt.plot(epochs, history_data[\"val_loss\"], label=\"Valid loss\")\nplt.legend()\n\nplt.subplot(142)\nplt.plot(epochs, history_data[\"mrcnn_class_loss\"], label=\"Train class loss\")\nplt.plot(epochs, history_data[\"val_mrcnn_class_loss\"], label=\"Valid class loss\")\nplt.legend()\n\nplt.show()\n\nplt.figure(figsize=(40,8))\n\nplt.subplot(141)\nplt.plot(epochs, history_data[\"mrcnn_bbox_loss\"], label=\"Train box loss\")\nplt.plot(epochs, history_data[\"val_mrcnn_bbox_loss\"], label=\"Valid box loss\")\nplt.legend()\n\nplt.subplot(142)\nplt.plot(epochs, history_data['mrcnn_mask_loss'], label=\"Train mask loss\")\nplt.plot(epochs, history_data['val_mrcnn_mask_loss'], label=\"Valid mask loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_epoch = np.argmin(history['val_loss'])\nprint(\"Best epoch: \", best_epoch+1, history['val_loss'][best_epoch])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#select trained model\nglob_list = glob.glob(f'/kaggle/input/head-saved-weights/mask_rcnn_fashion2020_{(best_epoch+1):04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InferenceConfig(FashionImagesConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\ninference_config = InferenceConfig()\n\n#Recreate the model in inference mode\nmodel = modellib.MaskRCNN(mode = 'inference',\n                         config = inference_config,\n                         model_dir = workDir)\n\n#Load trained weights \nassert model_path != \"\", \"Provide path to trained weights\"\nprint(\"Loading weights from \", model_path)\nmodel.load_weights(model_path, by_name = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_data = sample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert data to run-length encoding\ndef to_rle(bits):\n    rle = []\n    pos = 0\n    for bit, group in itertools.groupby(bits):\n        group_list = list(group)\n        if bit:\n            rle.extend([pos, sum(group_list)])\n        pos += len(group_list)\n    return rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fix overlapped masks\ndef fix_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis = 0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype = bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_position = np.where(masks[:, :, m] == True)\n        if np.any(mask_position):\n            y1, x1 = np.min(mask_position, axis = 1)\n            y2, x2 = np.max(mask_position, axis = 1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation = cv2.INTER_AREA)  \n    return img","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"\"\"\"%%time\nsubmission_list = []\nmissing_count = 0\n\nfor i, row in tqdm(sample_data.iterrows(), total = len(sample_data)):\n    image = resize_image(str(dataDir + '/test/' + row['ImageId']) + '.jpg')\n    result = model.detect([image])[0]\n    if result['masks'].size > 0:\n        masks, _ = fix_masks(result['masks'], result['rois'])\n        for m in range(masks.shape[-1]):\n            mask = masks[:, :, m].ravel(order = 'F')\n            rle = to_rle(mask)\n            label = result['class_ids'][m] - 1\n            submission_list.append([row['ImageId'], ' '.join(list(map(str, rle))), label, np.NaN])\n    else:\n        # The system does not allow missing ids\n        submission_list.append([row['ImageId'], '1 1', 23, np.NaN])\n        missing_count += 1\n    \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"validation_pred_df = pd.DataFrame(submission_list)\nvalidation_pred_df.columns = ['ImageId', 'EncodedPixels', 'ClassId']\nvalidation_pred_df = validation_pred_df.groupby('ImageId')['EncodedPixels', 'ClassId'].agg(lambda x: list(x))\n\nImageId = pd.Series(validation_pred_df.index)\nvalidation_pred_df.index = pd.Index(list(range(len(validation_pred_df))))\nvalidation_pred_df['ImageId'] = ImageId\n\nvalidation_pred_df\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample_submission.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"submission_data = pd.DataFrame(submission_list, columns=sample_submission.columns.values)\nprint(\"Total image results: \", submission_data['ImageId'].nunique())\nprint(\"Missing Images: \", missing_count)\nsubmission_data.head()\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#submission_data.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_raw = pd.read_csv(\"/kaggle/input/submissionraw/submission_non_grouped.csv\")\nsubmission_raw.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = submission_raw.groupby('ImageId')['EncodedPixels', 'ClassId', 'AttributesIds'].agg(lambda x: list(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}