{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom pathlib import Path\n#import seaborn as sns\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n\nfrom PIL import Image, ImageDraw, ImageFont\n#import cv2\n\nimport regex as re\nimport math\nimport random\n\nfrom itertools import compress\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\n\nimport torchvision\nimport torchvision.transforms as transforms\nfrom tqdm import tnrange, tqdm_notebook\n\nfrom sklearn.manifold import TSNE\nfrom sklearn.decomposition import TruncatedSVD\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torchvision.models.detection.rpn import RPNHead\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\npath = Path(\"/kaggle/input/kuzushiji-recognition\")\n#path = Path(\"kuzushiji-recognition\")\n\ntrain = pd.read_csv(path/\"train.csv\")\nunicode = pd.read_csv(path/\"unicode_translation.csv\")\nsubmission = pd.read_csv(path/\"sample_submission.csv\")\n\ntrain = train.dropna(axis = 0, how ='any') \n\ntrain_dir = Path(path/\"train_images\")\ntest_dir = Path(path/\"test_images\")\n\ndef toPath(string):\n    if \".jpg\" not in string:\n        string = string + \".jpg\"\n    return string\n\ndef toID(string):\n    if string[-4:] ==\".jpg\":\n        string = string[:-4]\n    return string\n\ndef displayImage(image):\n    plt.figure(figsize=(15,15))\n    this_img = Image.open(train_dir/toPath(image))\n    plt.imshow(this_img)\n    return plt\n\ndef getImageArray(image):\n    this_img = Image.open(train_dir/toPath(image))\n    return plt\n\ndef drawBoxAndText(ax, label):\n    codepoint, x, y, w, h = label\n    x, y, w, h = int(x), int(y), int(w), int(h)\n    rect = Rectangle((x, y), w, h, linewidth=1, edgecolor=\"r\", facecolor=\"none\")\n    ax.add_patch(rect)\n    ax.text(x, y - 20, getUnicode(codepoint),\n            fontproperties=prop,\n            color=\"r\",\n           size=16)\n    return ax\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\ndef getUnicode(code):\n    char = unicode.loc[unicode[\"Unicode\"] == code][\"char\"].values\n    if len(char) > 0:\n        return char[0]\n    else:\n        return '0'\n# Any results you write to the current directory are saved as output.\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n#device = torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def is_dist_avail_and_initialized():\n    if not dist.is_available():\n        return False\n    if not dist.is_initialized():\n        return False\n    return True\n\n\ndef get_world_size():\n    if not is_dist_avail_and_initialized():\n        return 1\n    return dist.get_world_size()\n\ndef reduce_dict(input_dict, average=True):\n    \"\"\"\n    Args:\n        input_dict (dict): all the values will be reduced\n        average (bool): whether to do average or sum\n    Reduce the values in the dictionary from all processes so that all processes\n    have the averaged results. Returns a dict with the same fields as\n    input_dict, after reduction.\n    \"\"\"\n    world_size = get_world_size()\n    if world_size < 2:\n        return input_dict\n    with torch.no_grad():\n        names = []\n        values = []\n        # sort the keys so that they are consistent across processes\n        for k in sorted(input_dict.keys()):\n            names.append(k)\n            values.append(input_dict[k])\n        values = torch.stack(values, dim=0)\n        dist.all_reduce(values)\n        if average:\n            values /= world_size\n        reduced_dict = {k: v for k, v in zip(names, values)}\n    return reduced_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# download a font that can display the characters\nimport matplotlib.font_manager as font_manager\n\nfontsize = 50\n\n# From https://www.google.com/get/noto/\n!wget -q --show-progress https://noto-website-2.storage.googleapis.com/pkgs/NotoSansCJKjp-hinted.zip\n!unzip -p NotoSansCJKjp-hinted.zip NotoSansCJKjp-Regular.otf > NotoSansCJKjp-Regular.otf\n!rm NotoSansCJKjp-hinted.zip\n\npath = './NotoSansCJKjp-Regular.otf'\nprop = font_manager.FontProperties(fname=path)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample id 100241706_00004_2\nimage_id = \"100241706_00004_2\"\nlabels = train[train[\"image_id\"] == image_id][\"labels\"].values[0]\nlabels_list = np.array(labels.split(\" \")).reshape(-1, 5)\n#labels_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"getUnicode(\"U+0031\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt = displayImage(image_id)\nax = plt.gca()\n\nfor label in labels_list:\n    ax = drawBoxAndText(ax, label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class KuzushijiDataset(object):\n    def __init__(self, df_data, root,  mode=\"train\"):\n        self.root = root\n        self.mode = mode\n        self.img = None\n        self.dataset = df_data\n        self.num_objs = num_classes\n        # self.transforms = transforms\n        # load all image files, sorting them to\n        # ensure that they are aligned\n        self.transform = transforms.ToTensor()\n        self.imgs = list(sorted(os.listdir(root)))\n    \n    def getMaskAndLabel(self, labels_list):\n        l_len = len(labels_list)\n        boxes = np.zeros((l_len, 4))\n        labels = np.zeros((l_len))\n        #print(ll)\n        #print(ll / label_length)\n        #label_tensor = torch.zeros((l_len, self.num_objs), dtype=torch.int64)\n        #for label in attribute_id.split():\n        for idx in range(l_len):\n            labels[idx] = unicode2labels[labels_list[idx][0]]\n            #labels = unicode2labels[labels_list[idx][0]]\n            #label_tensor[idx, labels] = 1\n            boxes[idx] = labels_list[idx][1:]\n            boxes[idx][2] = boxes[idx][2] + boxes[idx][0]\n            boxes[idx][3] = boxes[idx][3] + boxes[idx][1]\n        \n        #label_tensor[l_len-1, self.num_objs-1] = 1\n        #print(masks) # l\n        #print(labels) # l\n        return boxes, labels\n\n    def __getitem__(self, idx):\n        # load images ad masks\n        #print(idx)\n        #temp_dataset = self.dataset.iloc[[idx]]\n        temp_dataset = self.dataset[idx]\n        #print(temp_dataset)\n        #print(temp_dataset[\"image_id\"].values[0])\n        #print(\"-------------------------------------------------------\")\n        \n        img_path = os.path.join(self.root, toPath(temp_dataset[\"image_id\"].values[0]))\n        img = Image.open(img_path).convert(\"RGB\")        \n        \n        # print(np.array(img).shape)\n        \n        labels_col = temp_dataset[\"labels\"].values[0]\n        labels_list = np.array(str(labels_col).split(\" \")).reshape(-1, 5)\n        l_len = len(labels_list)\n        \n        boxes, labels = self.getMaskAndLabel(labels_list)        \n        masks = np.zeros((np.array(img).shape[0], np.array(img).shape[1]))\n        # print(masks.shape)\n        for i in range(l_len):\n            x, y, w, h = boxes[i]\n            x, y, w, h = int(x), int(y), int(w), int(h)\n            masks[y:h, x:w] = 1\n            #masks[x:w, y:h] = 1\n            #print(masks[x:h, y:w])\n        \n        image_id = torch.tensor([idx])\n        # suppose all instances are not crowd\n        \n        labels = torch.as_tensor(labels, dtype=torch.int64)\n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        masks = torch.as_tensor(masks, dtype=torch.float32)\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        iscrowd = torch.zeros((self.num_objs,), dtype=torch.int64)\n\n        target = {}\n        target[\"boxes\"] = boxes\n        target[\"labels\"] = labels\n        target[\"masks\"] = masks\n        target[\"image_id\"] = image_id\n        target[\"area\"] = area\n        target[\"iscrowd\"] = iscrowd\n    \n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, target\n\n    def __len__(self):\n        return len(self.imgs)\n\"\"\"\nkd = KuzushijiDataset(train_dir)\nimg, target = kd.__getitem__(0)\n\nim = Image.fromarray(target[\"masks\"])\n#im = im.convert('RGB')\nplt.figure(figsize=(15,15))\nplt.imshow(im, cmap='gray')\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_instance_segmentation(num_classes):\n    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n    # create an anchor_generator for the FPN\n    # which by default has 5 outputs\n    \n    anchor_generator = AnchorGenerator(sizes=((8, 16, 32, 64, 128),),aspect_ratios=((0.5, 1.0, 2.0),))\n    #print(anchor_generator)\n    #model.rpn.anchor_generator = anchor_generator\n\n    # 256 because that's the number of features that FPN returns\n    #model.rpn.head = RPNHead(256, anchor_generator.num_anchors_per_location()[0])\n    # replace the pre-trained head with a new one\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nunicode_map = {codepoint: char for codepoint, char in unicode.values}\nunicode2labels = dict(zip(unicode_map.keys(), range(len(unicode_map.keys()))))\n\n# our dataset has two classes only - background and Kuzushiji unicode\nnum_classes = len(unicode_map.keys()) + 1\n# use our dataset and defined transformations\ndataset = KuzushijiDataset(train, train_dir, mode=\"train\")\n#dataset_valid = KuzushijiDataset(train, train_dir, mode=\"train\")\n#dataset_test = KuzushijiDataset(train, test_dir, mode=\"test\")\n\n#indices = torch.randperm(len(dataset)).tolist()\n#dataset = torch.utils.data.Subset(dataset, indices)\n#dataset_valid = torch.utils.data.Subset(dataset, indices)\n\ndata_loader = torch.utils.data.DataLoader(\n    dataset, batch_size=4, shuffle=True, num_workers=1, collate_fn=collate_fn)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hello","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model_instance_segmentation(num_classes)\nmodel.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.0001,momentum=0.9, weight_decay=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 1\nmodel_file = \"model_saved.pkl\"\n\nmodel.train()\nfor epoch in range(num_epochs):\n    final_loss_value = np.inf\n    total_loss_value = 0\n    loss_value = 0\n    losses = 0\n    # train for one epoch, printing every 10 iterations\n    for batch_idx, (images, target) in enumerate(data_loader):\n        images = list(image.to(device) for image in images)\n        target = [{k: v.to(device) for k, v in t.items()} for t in target]\n        loss_dict = model(images, target)\n        #print(batch_idx)\n        #print(loss_dict['loss_classifier'])\n        losses = sum(loss for loss in loss_dict.values())\n        loss_dict_reduced = reduce_dict(loss_dict)\n        losses_reduced = sum(loss for loss in loss_dict_reduced.values())\n        #print(loss_dict_reduced)\n        #loss_value =+ losses_reduced.item()\n        loss_value += losses_reduced.item() * len(images)\n    \n    loss_value = loss_value / len(data_loader.dataset)\n    #train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=4)\n    optimizer.zero_grad()\n    losses.backward()\n    optimizer.step()\n    # update the learning rate\n    #lr_scheduler.step()\n    if loss_value < final_loss_value:\n        final_loss_value = loss_value\n        with open(model_file, \"wb\") as f:\n            torch.save(model.state_dict(), f)\n            print(\"Model saved :\", model_file)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":1}