{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n!mkdir train_images_mask\nprint(os.listdir(\"../input/kuzushiji-recognition\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/kuzushiji-recognition/train.csv\")\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's take a look at the first image"},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nfrom skimage.io import imread\nfrom skimage.color import rgb2gray\n\n\nexample_file = glob.glob(\"../input/kuzushiji-recognition/train_images/{}.jpg\".format(df.iloc[0,0]))[0]\nim = imread(example_file,as_gray=True)\n\nplt.figure(figsize=(10,10))\nplt.imshow(im,cmap='gray')\n\n#print(\"../input/train_images/{}.jpg\".format(df.iloc[0,0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The character locations will be stored in char_locs. These are in the form of x,y,width,height."},{"metadata":{"trusted":true},"cell_type":"code","source":"char_locs = df.iloc[0,1].split()\n\n# Reshape into matrix\nchar_locs = np.reshape(np.asarray(char_locs), (len(char_locs)//5, 5))\nchar_unicode = char_locs[:,0]\nchar_locs = char_locs[:,1:].astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we'll take a look at the characters in more detail."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(20,10))\nplt.subplot(1,4,1)\nplt.imshow(im,'gray')\n\nfor i in np.arange(char_locs.shape[0]):\n\n    x = char_locs[i,0]\n    y = char_locs[i,1]\n    w = char_locs[i,2]\n    h = char_locs[i,3]\n\n    plt.plot([x,x+w],[y,y],'r')\n    plt.plot([x,x+w],[y+h,y+h],'r')\n    plt.plot([x,x],[y,y+h],'r')\n    plt.plot([x+w,x+w],[y,y+h],'r')\n    \nr_samps = np.random.randint(0,char_locs.shape[0],9)\nsps = [2,3,4,6,7,8,10,11,12]\nfor i in np.arange(0,r_samps.shape[0]):\n    plt.subplot(3,4,sps[i])\n    x = char_locs[r_samps[i],0]\n    y = char_locs[r_samps[i],1]\n    w = char_locs[r_samps[i],2]\n    h = char_locs[r_samps[i],3]\n    plt.imshow(im[y:y+h,x:x+w],cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now to detect the characters\n# inspired by https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Masking\nHere we will get rid of some of the noise and make the characters binary - a 1 for the space occupied by the character and a 0 otherwise. Take a look at this example to see what I mean."},{"metadata":{"trusted":true},"cell_type":"code","source":"x = char_locs[50,0]\ny = char_locs[50,1]\nw = char_locs[50,2]\nh = char_locs[50,3]\n\nz_im = (im[y:y+h,x:x+w]-np.mean(im[y:y+h,x:x+w]))/np.std(im[y:y+h,x:x+w])\n\nplt.figure(figsize=(10,10))\nplt.subplot(1,2,1)\nplt.imshow(z_im,cmap='gray', vmin=np.amin(z_im), vmax=np.amax(z_im))\nplt.subplot(1,2,2)\nim_mask = np.copy(im[y:y+h,x:x+w])\nim_mask[z_im>0] = 0\nim_mask[z_im<=0] = 1\nplt.imshow(im_mask,cmap='gray',vmin=0,vmax=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we'll do it for every character and \"binarize\" the entire image, saving it as a file."},{"metadata":{"trusted":true},"cell_type":"code","source":"im_mask_whole = np.zeros((im.shape[0],im.shape[1]))\n\nfor i in np.arange(0,char_locs.shape[0]):\n    x = char_locs[i,0]\n    y = char_locs[i,1]\n    w = char_locs[i,2]\n    h = char_locs[i,3]\n\n    z_im = (im[y:y+h,x:x+w]-np.mean(im[y:y+h,x:x+w]))/np.std(im[y:y+h,x:x+w])\n\n    im_mask = np.copy(im[y:y+h,x:x+w])\n    im_mask[z_im>0] = 0\n    im_mask[z_im<=0] = 1\n    \n    im_mask_whole[y:y+h,x:x+w] = im_mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.imshow(im_mask_whole,cmap='gray',vmin=0,vmax=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import imageio\nimageio.imwrite('./train_images_mask/mask_{}.jpg'.format(df.iloc[0,0]), im_mask_whole)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's make that a function so we can do it for all of the training images. This will take a while and Kaggle doesn't allow me to write a bunch of files out and save the kernel, so I've commented out the save below."},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_masks(filename_df):\n    \n    for i in np.arange(0,filename_df.shape[0]):\n        # Load the file\n        file = glob.glob(\"../input/kuzushiji-recognition/train_images/{}.jpg\".format(df.iloc[i,0]))[0]\n        im = imread(file,as_gray=True)\n        # Make im_mask_whole to store the masked image\n        im_mask_whole = np.zeros((im.shape[0],im.shape[1]))\n        # If the image is not NaN, continue\n        if not(pd.isnull(filename_df.iloc[i,1])):\n            \n            # Make a list of the character names (unicode) and locations\n            char_locs = filename_df.iloc[i,1].split()\n            # Reshape into matrix\n            char_locs = np.reshape(np.asarray(char_locs), (len(char_locs)//5, 5))\n            char_unicode = char_locs[:,0]\n            # Locations are integers\n            char_locs = char_locs[:,1:].astype(np.int)\n        \n            for j in np.arange(0,char_locs.shape[0]):\n                x = char_locs[j,0]\n                y = char_locs[j,1]\n                w = char_locs[j,2]\n                h = char_locs[j,3]\n\n                z_im = (im[y:y+h,x:x+w]-np.mean(im[y:y+h,x:x+w]))/np.std(im[y:y+h,x:x+w])\n\n                im_mask = np.copy(im[y:y+h,x:x+w])\n                im_mask[z_im>0] = 0\n                im_mask[z_im<=0] = 1\n    \n                im_mask_whole[y:y+h,x:x+w] = im_mask\n        \n            #imageio.imwrite('./train_images_mask/mask_{}.jpg'.format(df.iloc[i,0]), im_mask_whole)\n        \n#make_masks(df)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Spot check the masks to make sure they look OK."},{"metadata":{"trusted":true},"cell_type":"code","source":"#file = glob.glob(\"../input/kuzushiji-recognition/train_images/{}.jpg\".format(df.iloc[200,0]))[0]\n#file_mask = glob.glob(\"./train_images_mask/mask_{}.jpg\".format(df.iloc[200,0]))[0]\n#im = imread(file,as_gray=True)\n#im_mask = imread(file_mask,as_gray=True)\n\n#plt.figure(figsize=(10,10))\n#plt.subplot(1,2,1)\n#plt.imshow(im,cmap='gray')\n#plt.subplot(1,2,2)\n#plt.imshow(im_mask,cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TorchVision\nCompletely ripped off from https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html\n** still in progress **"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport torch.utils.data\nfrom PIL import Image\n\nclass KuzushijiDataset(torch.utils.data.Dataset):\n    def __init__(self, root, transforms=None):\n        self.root = root\n        self.transforms = transforms\n        # load all image files, sorting them to\n        # ensure that they are aligned\n        self.imgs = list(sorted(os.listdir(os.path.join(root, \"../input/kuzushiji-recognition/train_images\"))))\n        self.masks = list(sorted(os.listdir(os.path.join(root, \"train_images_mask\"))))\n\n    def __getitem__(self, idx):\n        # load images ad masks\n        img_path = os.path.join(self.root, \"../input/kuzushiji-recognition/train_images\", self.imgs[idx])\n        mask_path = os.path.join(self.root, \"train_images_mask\", self.masks[idx])\n        img = Image.open(img_path).convert(\"RGB\")\n        # note that we haven't converted the mask to RGB,\n        # because each color corresponds to a different instance\n        # with 0 being background\n        mask = Image.open(mask_path)\n\n        mask = np.array(mask)\n        # instances are encoded as different colors\n        obj_ids = np.unique(mask)\n        # first id is the background, so remove it\n        obj_ids = obj_ids[1:]\n\n        # split the color-encoded mask into a set\n        # of binary masks\n        masks = mask == obj_ids[:, None, None]\n\n        # get bounding box coordinates for each mask\n        num_objs = len(obj_ids)\n        boxes = []\n        for i in range(num_objs):\n            pos = np.where(masks[i])\n            xmin = np.min(pos[1])\n            xmax = np.max(pos[1])\n            ymin = np.min(pos[0])\n            ymax = np.max(pos[0])\n            boxes.append([xmin, ymin, xmax, ymax])\n\n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        # there is only one class\n        labels = torch.ones((num_objs,), dtype=torch.int64)\n        masks = torch.as_tensor(masks, dtype=torch.uint8)\n\n        image_id = torch.tensor([idx])\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)\n\n        target = {}\n        target[\"boxes\"] = boxes\n        target[\"labels\"] = labels\n        target[\"masks\"] = masks\n        target[\"image_id\"] = image_id\n        target[\"area\"] = area\n        target[\"iscrowd\"] = iscrowd\n\n        if self.transforms is not None:\n            img, target = self.transforms(img, target)\n\n        return img, target\n\n    def __len__(self):\n        return len(self.imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection.mask_rcnn import MaskRCNNPredictor\n\n\ndef get_model_instance_segmentation(num_classes):\n    # load an instance segmentation model pre-trained pre-trained on COCO\n    model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)\n\n    # get number of input features for the classifier\n    in_features = model.roi_heads.box_predictor.cls_score.in_features\n    # replace the pre-trained head with a new one\n    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n    # now get the number of input features for the mask classifier\n    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels\n    hidden_layer = 256\n    # and replace the mask predictor with a new one\n    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,\n                                                       hidden_layer,\n                                                       num_classes)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I will do the rest offline and see if it works since I need to import some scripts etc."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}