{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport fastai\nfrom fastai.vision import *\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def open_mask_rle(mask_rle:str, shape:Tuple[int, int])->ImageSegment:\n    \"Return `ImageSegment` object create from run-length encoded string in `mask_lre` with size in `shape`.\"\n    x = FloatTensor(rle_decode(str(mask_rle), shape).astype(np.uint8))\n    x = x.view(shape[1], shape[0], -1)\n    return ImageSegment(x.permute(2,1,0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zzip = list(zip(trn['ImageId'].values, trn['Height'].values, trn['Width'].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"msk = list(trn['EncodedPixels'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m1 = open_mask_rle(msk[0], (zzip[0][1], zzip[0][2]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = open_image('../input/train/'+ zzip[0][0])\n_,axs = plt.subplots(1,3, figsize=(16,8))\nimg.show(ax=axs[0], title='no mask')\nimg.show(ax=axs[1], y=m1, title='masked')\nm1.show(ax=axs[2], title='mask only', alpha=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_value_counts = trn.ClassId.value_counts().reset_index()\ndf_train_value_counts.columns=['ClassId', 'Count']\ndf_sample_unique = trn.groupby('ClassId', group_keys=False).apply(lambda df: df.sample(1))\ndf_sample_unique_counts = df_sample_unique.merge(df_train_value_counts, on='ClassId')\ndf_sample_unique_counts.sort_values(by='Count', ascending=False, inplace=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RleSegList(ImageList):\n    def __init__(self, items, itemsB=None, **kwargs):\n        super().__init__(items, **kwargs)\n        self.itemsB = itemsB\n        self.copy_new.append('itemsB')\n    def get(self, i):\n        img1 = super().get(i)\n        fn = self.itemsB[random.randint(0, len(self.itemsB)-1)]\n        return ImageTuple(img1, open_image(fn))\n    @classmethod\n    def from_df(cls, path, folderA, folderB, **kwargs):\n        itemsB = ImageList.from_folder(path/folderB).items\n        res = super().from_folder(path/folderA, itemsB=itemsB, **kwargs)\n        res.path = path\n        return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zls = list(zip(trn['EncodedPixels'].values, trn['Height'].values, trn['Width'].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epls,hls,wls = list(trn['EncodedPixels']), list(trn['Height']), list(trn['Width'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"qid = list(set(trn['ImageId']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ImageId = qid[6] #'00000663ed1ff0c4e0132b9b9ac53f6e.jpg'\n\nimg = plt.imread('../input/train/' + ImageId)\nimg_masks = trn.loc[trn['ImageId'] == ImageId, 'EncodedPixels'].tolist()\nshape = (list(trn.loc[trn['ImageId'] == ImageId, 'Height'])[0], \n         list(trn.loc[trn['ImageId'] == ImageId, 'Width'])[0])\n\n\nall_masks = np.zeros((shape))\nfor mask in img_masks:\n    amasks = open_mask_rle(mask, (shape))\n    all_masks += amasks.px.numpy().squeeze()\n\nfig, axarr = plt.subplots(1, 3, figsize=(20, 45))\naxarr[0].axis('off')\naxarr[1].axis('off')\naxarr[2].axis('off')\naxarr[0].imshow(img)\naxarr[1].imshow(all_masks)\naxarr[2].imshow(img)\naxarr[2].imshow(all_masks, alpha=0.4)\nplt.tight_layout(h_pad=0.1, w_pad=0.1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trfm = transforms.Compose([\n            #transforms.RandomResizedCrop(224),\n            #transforms.RandomHorizontalFlip(),\n            transforms.ToTensor(),\n            #normalize,\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = albumentations.Compose([\n    transforms.ToTensor()\n    ])\ndata_transforms_test = albumentations.Compose([\n    albumentations.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    transforms.ToTensor()\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tes = pd.read_csv('../input/sample_submission.csv')\ntes.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Imat(Dataset):\n    def __init__(self, df, datafolder, datatype='train', \n                 transform = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor()])):\n        self.datafolder = datafolder\n        self.df = df\n        self.datatype = datatype\n        self.image_files_list = list(self.df['ImageId'])\n        if self.datatype == 'train':\n            self.mask_list = list(self.df['EncodedPixels'])\n            self.h = list(self.df['Height'])\n            self.w = list(self.df['Width'])\n        self.transform = transform\n        \n        if self.datatype == 'train':\n            self.labels = [np.float32(i) for i in list(self.df['ClassId'])]\n        else:\n            self.labels = [np.float32(0.0) for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        img = open_image(img_name)\n        if self.datatype == 'train':\n            mask = open_mask_rle(self.mask_list[idx], (self.h[idx], self.w[idx]))\n            image = img.px\n            label = self.labels[idx]\n            return image, mask.px, label\n        else:\n            img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n            img = open_image(img_name)\n            image = img.px\n            label = [np.float32(0.0) for _ in range(len(self.image_files_list))]\n            label = label[idx]\n            return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vlx = trn.sample(frac=0.2).index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = trn.loc[~trn.index.isin(vlx)].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vl = trn.loc[trn.index.isin(vlx)].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trds = Imat(tr, datafolder='../input/train', datatype='train', transform=tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vrds = Imat(vl, datafolder='../input/train', datatype='train', transform=tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"teds = Imat(tes, datafolder='../input/test', datatype='test', transform=tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter(trds))[1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = DataBunch.create(trds, vrds, teds, bs=8, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"next(iter(data.train_dl))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#https://github.com/lmoroney/dlaicourse/blob/master/Exercises/Exercise%205%20-%20Real%20World%20Scenarios/Exercise%205%20-%20Answer.ipynb\n#https://cmdlinetips.com/2018/12/how-to-loop-through-pandas-rows-or-how-to-iterate-over-pandas-rows/\n%matplotlib inline\n\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\n\n# Parameters for our graph; we'll output images in a 4x4 configuration\nnrows = 12\nncols = 4\n\npic_index = 0 # Index for iterating over images\n# Set up matplotlib fig, and size it to fit 4x4 pics\nfig = plt.gcf()\nfig.set_size_inches(ncols*4, nrows*4)\n\nctr=0\nfor index, row in df_sample_unique_counts.head(n=24).iterrows():\n     #print(index, row)\n          # Set up subplot; subplot indices start at 1\n    img_path = row.ImageId\n    classId = row.ClassId\n    rle = row.EncodedPixels\n    h = row.Height\n    w = row.Width\n    mask = open_mask_rle(rle, (h, w))\n    mk = np.array(mask.px).reshape(mask.shape[1], mask.shape[2], -1).astype(np.uint8)\n    ctr+=1\n    sp = plt.subplot(nrows, ncols, ctr)\n    sp.axis('Off') # Don't show axes (or gridlines)\n    img_path = '../input/train/' + img_path\n    img = mpimg.imread(img_path)\n    dimg = np.dstack((img, mk))\n    plt.imshow(img)\n    plt.imshow(dimg[..., -1])\n    plt.title(classId)\n    \nfor index, row in df_sample_unique_counts.head(n=24).iterrows():\n     #print(index, row)\n          # Set up subplot; subplot indices start at 1\n    img_path = row.ImageId\n    classId = row.ClassId\n    rle = row.EncodedPixels\n    h = row.Height\n    w = row.Width\n    mask = open_mask_rle(rle, (h, w))\n    mk = np.array(mask.px).reshape(mask.shape[1], mask.shape[2], -1).astype(np.uint8)\n    ctr+=1\n    sp = plt.subplot(nrows, ncols, ctr)\n    sp.axis('Off') # Don't show axes (or gridlines)\n    img_path = '../input/train/' + img_path\n    img = mpimg.imread(img_path)\n    dimg = np.dstack((img, mk))\n    plt.imshow(img)\n    plt.title(classId)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}