{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **ARTIFICIAL INTELLIGENCE J-Component**\n\n**Team Members: **\n\nB. VISHNU CHARAN -17BCI0090\n\nVARUN RAJ -17BCE0246\n\n**Project Title: **\n\niMaterialist (Fashion) 2019 at FGVC6\n\n**Slot:** \n\nF2+TF2\n\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # basic linear algebra and array operations\nimport pandas as pd # used for data processing,eg for I/O of a CSV file. \nimport os  # for the path operations\nprint(os.listdir(\"../input\"))\nimport cv2   # cv2 is used for image processing operations\nimport matplotlib.pyplot as plt  # matplotlib is used for data visualization\n%matplotlib inline\nfrom tqdm import tqdm_notebook as tqdm  # tqdm is used for visualizing the progress of running\n\nimport torch # Contains data structures for multi-dimensional tensors and mathematical operations.\nimport torch.nn as nn # A kind of Tensor, to be considered a module parameter in torch\nfrom torch import optim # A package implementing various optimization algorithms\nimport torchvision.transforms as transforms  # Used for common image transformations.\nimport torch.nn.functional as F  # Used for Convolution Functions\nfrom torch.autograd import Function, Variable  # Automatic differentiation of arbitrary scalar valued functions.\nfrom pathlib import Path # For getting the path \nfrom itertools import groupby # To group the elements","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_dir = \"../input/imaterialist-fashion-2019-FGVC6/\"\ntrain_img_dir = \"../input/imaterialist-fashion-2019-FGVC6/train/\"\ntest_img_dir = \"../input/imaterialist-fashion-2019-FGVC6/test/\"\n\nWIDTH = 512\nHEIGHT = 512\ncategory_num = 47\n\nratio = 8\n\nepoch_num = 2\nbatch_size = 4\n\ndevice = \"cuda:0\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir(\"../input/imaterialist-fashion-2019-FGVC6/train/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir(test_img_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(input_dir + \"train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\nos.chdir('Mask_RCNN')\n\n!rm -rf .git # To prevent an error when the kernel is committed\n!rm -rf images assets # To prevent displaying images at the bottom of a kernel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(str('/kaggle/working/Mask_RCNN'))\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For demonstration purpose, the classification ignores attributes (only categories),\n# and the image size is set to 512, which is the same as the size of submission masks\n\nNUM_CATS = 46\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionConfig(Config):\n    NAME = \"fashion\"\n    NUM_CLASSES = NUM_CATS + 1 \n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 4 # a memory error occurs when IMAGES_PER_GPU is too high\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    STEPS_PER_EPOCH = 1000\n    VALIDATION_STEPS = 200\n    \nconfig = FashionConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nwith open(\"/kaggle/input/imaterialist-fashion-2019-FGVC6/label_descriptions.json\") as f:\n    label_descriptions = json.load(f)\n\nlabel_names = [x['name'] for x in label_descriptions['categories']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(label_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(label_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attribute_names = [x['name'] for x in label_descriptions['attributes']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(attribute_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(attribute_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(label_descriptions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df = pd.read_csv(\"/kaggle/input/imaterialist-fashion-2019-FGVC6/train.csv\")\n\nmultilabel_percent = len(segment_df[segment_df['ClassId'].str.contains('_')])/len(segment_df)*100\nprint(f\"Segments that have attributes: {multilabel_percent:.2f}%\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df['CategoryId'] = segment_df['ClassId'].str.split('_').str[0]\n\nprint(\"Total segments: \", len(segment_df))\nsegment_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df = segment_df.groupby('ImageId')['EncodedPixels', 'CategoryId'].agg(lambda x: list(x))\nsize_df = segment_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(image_path):\n    try:\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n    except Exception as e:\n        pass\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nDATA_DIR = Path('/kaggle/input/imaterialist-fashion-2019-FGVC6')\nROOT_DIR = Path('/kaggle/working/imaterialist-fashion-2019-FGVC6')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FashionDataset(utils.Dataset):\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Adding the  classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", i+1, name)\n        \n        # Add the images \n        for i, row in df.iterrows():\n            self.add_image(\"fashion\", image_id=row.name, path=str(DATA_DIR/'train'/row.name), labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], height=row['Height'], width=row['Width'])\n    \n    # This function returns the path and the label_names of the image\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n    # This function is used to resize the image\n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    # This function is used to generate a mask for the given image\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndataset = FashionDataset(image_df)\ndataset.prepare()\n\nfor i in range(6):\n    image_id = random.choice(dataset.image_ids)\n    print(dataset.image_reference(image_id))\n    \n    image = dataset.load_image(image_id)\n    mask, class_ids = dataset.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Utility Functions**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Used to perform one hot encoding of the categorical variables.\n\ndef make_onehot_vec(x):\n    vec = np.zeros(category_num)\n    vec[x] = 1\n    return vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This function is used to create a mask of the costumes which are in the image dataset\ndef make_mask_img(segment_df):\n    seg_width = segment_df.at[0, \"Width\"]\n    seg_height = segment_df.at[0, \"Height\"]\n    seg_img = np.full(seg_width*seg_height, category_num-1, dtype=np.int32)\n    for encoded_pixels, class_id in zip(segment_df[\"EncodedPixels\"].values, segment_df[\"ClassId\"].values):\n        pixel_list = list(map(int, encoded_pixels.split(\" \")))\n        for i in range(0, len(pixel_list), 2):\n            start_index = pixel_list[i] - 1\n            index_len = pixel_list[i+1] - 1\n            seg_img[start_index:start_index+index_len] = int(class_id.split(\"_\")[0])\n    seg_img = seg_img.reshape((seg_height, seg_width), order='F')\n    seg_img = cv2.resize(seg_img, (WIDTH, HEIGHT), interpolation=cv2.INTER_NEAREST)\n    return seg_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Utility function is used to generate the training dataset in a format which can be used while giving the \n# dataset to the traning model\ndef train_generator(df, batch_size):\n    img_ind_num = df.groupby(\"ImageId\")[\"ClassId\"].count() \n    index = df.index.values[0]\n    trn_images = []\n    seg_images = []\n    for i, (img_name, ind_num) in enumerate(img_ind_num.items()):\n        try:\n            img = cv2.imread(train_img_dir + img_name)\n            img = cv2.resize(img, (WIDTH, HEIGHT), interpolation=cv2.INTER_AREA)\n            segment_df = (df.loc[index:index+ind_num-1, :]).reset_index(drop=True)\n            index += ind_num\n            if segment_df[\"ImageId\"].nunique() != 1:\n                raise Exception(\"Index Range Error\")\n            seg_img = make_mask_img(segment_df)\n        \n            img = img.transpose((2, 0, 1))    \n            trn_images.append(img)\n            seg_images.append(seg_img)\n            if((i+1) % batch_size == 0):\n                yield np.array(trn_images, dtype=np.float32) / 255, np.array(seg_images, dtype=np.int32)\n                trn_images = []\n                seg_images = []\n        except Exception as e:\n            pass\n        if(len(trn_images) != 0):\n            yield np.array(trn_images, dtype=np.float32) / 255, np.array(seg_images, dtype=np.int32)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Utility function is used to generate the test dataset in a format which is same as the train dataset\ndef test_generator(df):\n    img_names = df[\"ImageId\"].values\n    for img_name in img_names:\n        try:\n            img = cv2.imread(test_img_dir + img_name)\n            img = cv2.resize(img, (WIDTH, HEIGHT), interpolation=cv2.INTER_AREA)\n            img = img.transpose((2, 0, 1))\n        except Exception as e:\n            pass\n        yield img_name, np.asarray([img], dtype=np.float32) / 255\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Utility function is used to encode the string\ndef encode(input_string):\n    return [(len(list(g)), k) for k,g in groupby(input_string)]\n\n# This Utility function is used to perform run length encoding\ndef run_length(label_vec):\n    encode_list = encode(label_vec)\n    index = 1\n    class_dict = {}\n    for i in encode_list:\n        if i[1] != category_num-1:\n            if i[1] not in class_dict.keys():\n                class_dict[i[1]] = []\n            class_dict[i[1]] = class_dict[i[1]] + [index, i[0]]\n        index += i[0]\n    return class_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Defining the Convoluional Neural Network and the UNET**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class double_conv(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super(double_conv, self).__init__()\n        self.conv = nn.Sequential(nn.Conv2d(in_ch, out_ch, 3, padding=1),nn.BatchNorm2d(out_ch),nn.ReLU(inplace=True),nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),nn.ReLU(inplace=True))\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\n\nclass inconv(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super(inconv, self).__init__()\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\n\nclass down(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super(down, self).__init__()\n        self.mpconv = nn.Sequential(nn.MaxPool2d(2),double_conv(in_ch, out_ch))\n\n    def forward(self, x):\n        x = self.mpconv(x)\n        return x\n\n\nclass up(nn.Module):\n    def __init__(self, in_ch, out_ch, bilinear=True):\n        super(up, self).__init__()\n\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        else:\n            self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)\n\n        self.conv = double_conv(in_ch, out_ch)\n\n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        diffX = x1.size()[2] - x2.size()[2]\n        diffY = x1.size()[3] - x2.size()[3]\n        x2 = F.pad(x2, (diffX // 2, int(diffX / 2),diffY // 2, int(diffY / 2)))\n        x = torch.cat([x2, x1], dim=1)\n        x = self.conv(x)\n        return x\n\n\nclass outconv(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super(outconv, self).__init__()\n        self.conv = nn.Conv2d(in_ch, out_ch, 1)\n\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\n    \nclass UNet(nn.Module):\n    def __init__(self, n_channels, n_classes):\n        super(UNet, self).__init__()\n        self.inc = inconv(n_channels, 64)\n        self.down1 = down(64, 128)\n        self.down2 = down(128, 256)\n        self.down3 = down(256, 512)\n        self.down4 = down(512, 512)\n        self.up1 = up(1024, 256)\n        self.up2 = up(512, 128)\n        self.up3 = up(256, 64)\n        self.up4 = up(128, 64)\n        self.outc = outconv(64, n_classes)\n\n    def forward(self, x):\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x5 = self.down4(x4)\n        x = self.up1(x5, x4)\n        x = self.up2(x, x3)\n        x = self.up3(x, x2)\n        x = self.up4(x, x1)\n        x = self.outc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training the dataset**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"333415 // 4  # Since it is 4 batches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.iloc[83348:83354, :]  # We will have a look at a small part of the dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.iloc[73350:73354, :]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The Final UNet Training**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"net = UNet(n_channels=3, n_classes=category_num).to(device)  #Trains a unet instance\noptimizer = optim.SGD(net.parameters(),lr=0.1,momentum=0.9,weight_decay=0.0005) # Optimizing the algorithm\ncriterion = nn.CrossEntropyLoss()  # It is useful when training a classification problem with a particular number of classes.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_sta = 73352\nval_end = 83351\ntrain_loss = []\nvalid_loss = []\nfor epoch in range(epoch_num):\n    epoch_trn_loss = 0\n    train_len = 0\n    net.train()\n    \n    # This is for training dataset\n    for iteration, (X_trn, Y_trn) in enumerate(tqdm(train_generator(train_df.iloc[:val_sta, :], batch_size))):\n        X = torch.tensor(X_trn, dtype=torch.float32).to(device)  #torch.Tensor is a multi-dimensional matrix containing elements of a single data type.\n        Y = torch.tensor(Y_trn, dtype=torch.long).to(device)\n        train_len += len(X)\n        \n        mask_pred = net(X)\n        loss = criterion(mask_pred, Y)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        epoch_trn_loss += loss.item()\n        \n        if iteration % 100 == 0:\n            print(\"train loss in {:0>2}epoch  /{:>5}iter:{:<10.8}\".format(epoch+1, iteration, epoch_trn_loss/(iteration+1)))\n        \n    train_loss.append(epoch_trn_loss/(iteration+1))\n    print(\"train {}epoch loss({}iteration):{:10.8}\".format(epoch+1, iteration, train_loss[-1]))\n    \n    \n    # This is for validation dataset\n    epoch_val_loss = 0\n    val_len = 0\n    net.eval()\n    for iteration, (X_val, Y_val) in enumerate(tqdm(train_generator(train_df.iloc[val_sta:val_end, :], batch_size))):\n        X = torch.tensor(X_val, dtype=torch.float32).to(device)\n        Y = torch.tensor(Y_val, dtype=torch.long).to(device)\n        val_len += len(X)\n            \n        mask_pred = net(X)\n        loss = criterion(mask_pred, Y)\n        epoch_val_loss += loss.item()\n        \n        if iteration % 100 == 0:\n            print(\"valid loss in {:0>2}epoch/{:>5}iter: {:<10.8}\".format(epoch+1, iteration, epoch_val_loss/(iteration+1)))\n        \n    valid_loss.append(epoch_val_loss/(iteration+1))\n    print(\"valid {}epoch loss({}iteration): {:10.8}\".format(epoch+1, iteration, valid_loss[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('/kaggle/input/imaterialist-fashion-2019-FGVC6/sample_submission.csv')\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Generating the predictions in the sub_list list\nsub_list = []\nnet.eval()\nfor img_name, img in test_generator(sample_df):\n    X = torch.tensor(img, dtype=torch.float32).to(device)\n    mask_pred = net(X)\n    mask_pred = mask_pred.cpu().detach().numpy()\n    mask_prob = np.argmax(mask_pred, axis=1)\n    mask_prob = mask_prob.ravel(order='F')\n    class_dict = run_length(mask_prob)\n    if len(class_dict) == 0:\n        sub_list.append([img_name, \"1 1\", 1])\n    else:\n        for key, val in class_dict.items():\n            sub_list.append([img_name, \" \".join(map(str, val)), key])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame(sub_list, columns=sample_df.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}