{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nimport pathlib\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\nimport tqdm","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\nsample_submission = pd.read_csv(\"../input/understanding_cloud_organization/sample_submission.csv\")\ntrain = pd.read_csv(\"../input/understanding_cloud_organization/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Transform the csv file to Image name and label\ntrain['Image_name'] = train['Image_Label'].apply(lambda x: x.split('_')[0])\ntrain['Label_name'] = train['Image_Label'].apply(lambda x: x.split('_')[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop('Image_Label',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.pivot('Image_name','Label_name','EncodedPixels')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ItemList creation"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = pathlib.Path('/kaggle/input/understanding_cloud_organization/')\npath_img = data_path/'train_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = (SegmentationItemList.\n            from_df(df=train.reset_index(),path=path_img,cols=\"Image_name\")\n            .split_by_rand_pct(0.2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiLabelImageSegment(ImageSegment):\n    \"\"\"Store overlapping masks in separate image channels\"\"\"\n\n    def show(self, ax:plt.Axes=None, figsize:tuple=(3,3), title:Optional[str]=None, hide_axis:bool=True,\n        cmap:str='tab20', alpha:float=0.5, class_names=None, **kwargs):\n        \"Show the masks on `ax`.\"\n             \n        # put all masks into a single channel\n        flat_masks = self.px[0:1, :, :].clone()\n        for idx in range(1, self.shape[0]): # shape CxHxW\n            mask = self.px[idx:idx+1, :, :] # slice tensor to a single mask channel\n            # use powers of two for class codes to keep them distinguishable after sum \n            flat_masks += mask * 2**idx\n        \n        # use same color normalization in image and legend\n        norm = matplotlib.colors.Normalize(vmin=0, vmax=2**self.shape[0]-1)\n        ax = show_image(Image(flat_masks), ax=ax, hide_axis=hide_axis, cmap=cmap, norm=norm,\n                        figsize=figsize, interpolation='nearest', alpha=alpha, **kwargs)\n        \n        # custom legend, see https://matplotlib.org/3.1.1/gallery/text_labels_and_annotations/custom_legends.html\n        cm = matplotlib.cm.get_cmap(cmap)\n        legend_elements = []\n        for idx in range(self.shape[0]):\n            c = 2**idx\n            label = class_names[idx] if class_names is not None else f\"class {idx}\"\n            line = Line2D([0], [0], color=cm(norm(c)), label=label, lw=4)\n            legend_elements.append(line)\n        ax.legend(handles=legend_elements)\n        \n        # debug info\n        # ax.text(10, 10, f\"px={self.px.size()}\", {\"color\": \"white\"})\n        \n        if title: ax.set_title(title)\n\n    def reconstruct(self, t:Tensor): \n        return MultiClassImageSegment(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Source: https://www.kaggle.com/keyurparalkar/multi-label-segmentation-using-fastai/\ndef bce_logits_floatify(input,target,reduction='mean'):\n    return F.binary_cross_entropy_with_logits(input,target.float(),reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiLabelSegmentationLabelList(SegmentationLabelList):\n    \"\"\"return a single image segment with all classes\"\"\"\n    def __init__(self, items:Iterator, src_img_size=None, classes:Collection = None, **kwargs):\n        super().__init__(items=items,classes=classes,**kwargs)\n        self.loss_func = bce_logits_floatify\n        self.src_img_size = src_img_size\n        self.copy_new += ['src_img_size']\n        \n    def open(self,rles):\n        masks = torch.zeros((len(self.classes),*self.src_img_size))  # shape CxHxW\n        for i, rle in enumerate(rles):\n            if(isinstance(rle,str)):\n                rle_to_mask = open_mask_rle(rle,self.src_img_size)\n                masks[i] = rle_to_mask.px.permute(0,2,1)\n        return MultiLabelImageSegment(masks)\n    \n    def analyze_pred(self,pred, thres:float=0.0):\n        #Binary masks\n        return (pred > thres).float()\n    \n    def reconstruct(self, t:Tensor):\n        return MultiLabelImageSegment(t)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = ['Fish','Flower','Gravel','Sugar'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mask_rle(img):\n    img = img.split(\"/\")[-1]    #get file name only\n    return train.loc[img, class_names].to_list()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reduce the image size into multiples of 4.\nimg_size = (84,132)\n\n#Train and test image sizes:\ntrain_img_dims = (1400, 2100) \n\nimg_size\n\nbatch_size=8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = [0,1,2,3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = item_list.label_from_func(func=get_mask_rle,label_cls=MultiLabelSegmentationLabelList,classes=classes,\n                                     src_img_size=train_img_dims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = item_list.add_test_folder(data_path/'test_images',label=\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 8\n\ntfms = ([],[])\nitem_list  = item_list.transform(tfms,tfm_y=True,size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (item_list.databunch(bs=batch_size)\n       .normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(rows=2,figsize=(15,10),class_names=class_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"# adapted from: https://www.kaggle.com/iafoss/unet34-dice-0-87\n# can use sigmoid on the input too, in this case the threshold would be 0.5\ndef dice_metric(pred, targs, threshold=0):\n    pred = (pred > threshold).float()\n    targs = targs.float()  # make sure target is float too\n    return 2.0 * (pred*targs).sum() / ((pred+targs).sum() + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn = unet_learner(data, models.resnet18, metrics=[dice_metric], wd = 1e-2).to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Set path for saving the model\nlearn.model_dir = '/kaggle/working/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10,max_lr=1e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('trained_model_fit1cyc',return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pydrive","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pydrive.drive import GoogleDrive","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pydrive.auth import GoogleAuth\n\ngauth = GoogleAuth()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drive = GoogleDrive(gauth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.show_results(imgsize=8, class_names=class_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#getting prediction on test dataset\na,b = learn.get_preds(ds_type=DatasetType.Test,with_loss=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.core.debugger import set_trace","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_masks(pred:Tensor, img_size=(4,350,525)) -> list:\n    for i in range(pred.shape[0]):\n#         set_trace()\n        mask = MultiLabelImageSegment(pred[i])\n        yield mask.resize(img_size)        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resized_preds = resize_masks(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_fnames = [str(fname).split('/')[-1] for fname in learn.data.test_ds.items]\ntest_fnames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Function for creating submission file:\ndef writeSubFile():\n    thres = 0 #Defining threshold for comparing it with masks\n    \n    with open('/kaggle/working/submission.csv','w') as f:\n        print('Writing submission file ...')\n        f.write('Image_Label,EncodedPixels\\n')\n        \n        for img_name, mask in zip(test_fnames, resize_masks(a)):\n            preds = mask.data > thres\n            for i,cls_name in enumerate(class_names):\n                rle = rle_encode(preds[i])\n                f.write(f\"{img_name}_{cls_name},{rle}\\n\")\n                \n    print('Submission file created ...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"writeSubFile()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('/kaggle/working/submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}