{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nfrom time import time\n\nimport numpy as np\nimport pandas as opd\n\nfrom tqdm import tqdm\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\n\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly as py\nimport plotly.graph_objs as go\nfrom plotly.subplots import make_subplots\n\ninit_notebook_mode(connected=True)\n\nimport warnings\nimport fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nfrom fastai.callbacks.hooks import *\nfrom fastai.utils.mem import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.utils.show_install import show_install; show_install()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fmt_now():\n    return datetime.today().strftime('%Y%m%d-%H%M%S')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input')\npath.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_img = path/'train_images'\n\nfnames_train = get_image_files(path_img)\nfor f in fnames_train[:3]:\n    print(f)\nprint('\\nTotal number of training images: {}'.format(len(fnames_train)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_test = path/'test_images'\n\nfnames_test = get_image_files(path_test)\nfor f in fnames_test[:3]:\n    print(f)\nprint('\\nTotal number of test images: {}'.format(len(fnames_test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_f = fnames_train[2]\nimg = open_image(img_f)\nimg.show(figsize=(10, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_img_label(img_lbl):\n    \"\"\"Return image and label from filename \"\"\"\n    s = img_lbl.split(\"_\")\n    assert len(s) == 2\n    return s[0], s[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{path}/train.csv')\ntrain['Image'] = train['Image_Label'].apply(lambda img_lbl: split_img_label(img_lbl)[0])\ntrain['Label'] = train['Image_Label'].apply(lambda img_lbl: split_img_label(img_lbl)[1])\ndel train['Image_Label']\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_with_mask = train.dropna(subset=['EncodedPixels'])\n\n#colors = ['gold', 'mediumturquoise', 'darkorange', 'lightgreen']\n\nfig = go.Figure(data=[go.Pie(labels=train_with_mask['Label'].value_counts().index, \n                             values=train_with_mask['Label'].value_counts().values)])\n\nfig.update_traces(hoverinfo=\"label+percent+name\")\n\nfig.update_layout(height=600, width=900, title = 'Class distribution')\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_counts = train.dropna(subset=['EncodedPixels']).groupby('Image')['Label'].nunique()\n\nfig = go.Figure()\n\nfig.add_trace(\n    go.Histogram(\n        x = class_counts,\n        xbins=dict(\n        start=0.5,\n        end=4.5,\n        size=1\n        ),\n    )\n)\n\nfig.update_layout(height=450, width=900, title = 'Distribution of no. labels per image')\n\nfig.update_layout(\n    xaxis_title_text='No. Image Class Labels', # xaxis label\n    yaxis_title_text='Count', # yaxis label\n    bargap=0.2, # gap between bars of adjacent location coordinates\n    bargroupgap=0.1 # gap between bars of the same location coordinates\n)\n\nfig.update_xaxes(tickvals=[1, 2, 3, 4])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.pivot(index='Image', columns='Label', values='EncodedPixels')\nassert len(train) == len(fnames_train)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Broken images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_img_fn(fname, figsize=(10, 10)):\n    img = open_image(fname)\n    img.show(figsize=figsize)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_img_info(fname):\n    show_img_fn(path_img/fname)\n    display(train.loc[[fname]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unusual_imgs = [\"1588d4c.jpg\", \"c0306e5.jpg\", \"c26c635.jpg\", \"fa645da.jpg\", \"41f92e5.jpg\", \"e5f2f24.jpg\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fname in unusual_imgs:\n    img = open_image(path_img/fname)\n    img.show(figsize=(5, 5), title=fname)     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Convert masks from/to RLE"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_dims = (1400, 2100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_to_mask(rle, shape):\n    mask_img = open_mask_rle(rle, shape)\n    mask = mask_img.px.permute(0, 2, 1)\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mask_to_rle(mask):\n    \"\"\" Convert binary 'mask' to RLE string \"\"\"\n    return rle_encode(mask.numpy().T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_mask_rle():\n    \"\"\" test case for mask RLE encode/decode\"\"\"\n    mask_rle = train.iloc[0]['Fish']\n    mask = rle_to_mask(mask_rle, train_img_dims)\n    mask_rle_enc = mask_to_rle(mask)\n    assert mask_rle_enc == mask_rle\n    \n    print(mask.shape)\n    Image(mask).show()\n    \ntest_mask_rle()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load images"},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = (SegmentationItemList\n            .from_df(df=train.reset_index(), path=path_img, cols='Image')\n             #.use_partial_data(sample_pct=0.1)\n            .split_by_rand_pct(0.2)\n            )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiLabelImageSegment(ImageSegment):\n    \"\"\"Store overlapping masks in separate image channels\"\"\"\n\n    def show(self, ax:plt.Axes=None, figsize:tuple=(3,3), title:Optional[str]=None, hide_axis:bool=True,\n        cmap:str='tab20', alpha:float=0.5, class_names=None, **kwargs):\n        \"Show the masks on `ax`.\"\n             \n        # put all masks into a single channel\n        flat_masks = self.px[0:1, :, :].clone()\n        for idx in range(1, self.shape[0]): # shape CxHxW\n            mask = self.px[idx:idx+1, :, :] # slice tensor to a single mask channel\n            # use powers of two for class codes to keep them distinguishable after sum \n            flat_masks += mask * 2**idx\n        \n        # use same color normalization in image and legend\n        norm = matplotlib.colors.Normalize(vmin=0, vmax=2**self.shape[0]-1)\n        ax = show_image(Image(flat_masks), ax=ax, hide_axis=hide_axis, cmap=cmap, norm=norm,\n                        figsize=figsize, interpolation='nearest', alpha=alpha, **kwargs)\n        \n     # custom legend, see https://matplotlib.org/3.1.1/gallery/text_labels_and_annotations/custom_legends.html\n        cm = matplotlib.cm.get_cmap(cmap)\n        legend_elements = []\n        for idx in range(self.shape[0]):\n            c = 2**idx\n            label = class_names[idx] if class_names is not None else f\"class {idx}\"\n            line = Line2D([0], [0], color=cm(norm(c)), label=label, lw=4)\n            legend_elements.append(line)\n        ax.legend(handles=legend_elements)\n        \n        # debug info\n        # ax.text(10, 10, f\"px={self.px.size()}\", {\"color\": \"white\"})\n        \n        if title: ax.set_title(title)\n\n    def reconstruct(self, t:Tensor): \n        return MultiClassImageSegment(t)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# source: https://forums.fast.ai/t/unet-how-to-get-4-channel-output/54674/4\ndef bce_logits_floatify(input, target, reduction='mean'):\n    return F.binary_cross_entropy_with_logits(input, target.float(), reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiLabelSegmentationLabelList(SegmentationLabelList):\n    \"\"\"Return a single image segment with all classes\"\"\"\n    # adapted from https://forums.fast.ai/t/how-to-load-multiple-classes-of-rle-strings-from-csv-severstal-steel-competition/51445/2\n    \n    def __init__(self, items:Iterator, src_img_size=None, classes:Collection=None, **kwargs):\n        super().__init__(items=items, classes=classes, **kwargs)\n        self.loss_func = bce_logits_floatify\n        self.src_img_size = src_img_size\n        # add attributes to copy by new() \n        self.copy_new += [\"src_img_size\"]\n    \n    def open(self, rles):        \n        # load mask at full resolution\n        masks = torch.zeros((len(self.classes), *self.src_img_size)) # shape CxHxW\n        for i, rle in enumerate(rles):\n            if isinstance(rle, str):  # filter out NaNs\n                masks[i] = rle_to_mask(rle, self.src_img_size)\n        return MultiLabelImageSegment(masks)\n    \n    def analyze_pred(self, pred, thresh:float=0.0):\n        # binarize masks\n        return (pred > thresh).float()\n    \n    \n    def reconstruct(self, t:Tensor): \n        return MultiLabelImageSegment(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = [\"Fish\", \"Flower\", \"Gravel\", \"Sugar\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masks_rle(img):\n    \"\"\"Get RLE-encoded masks for this image\"\"\"\n    img = img.split(\"/\")[-1]  # get filename only\n    return train.loc[img, class_names].to_list()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = (84, 132)  # use multiple of 4\nimg_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = [0, 1, 2, 3] # no need for a \"void\" class: if a pixel isn't in any mask, it is not labelled\nitem_list = item_list.label_from_func(func=get_masks_rle, label_cls=MultiLabelSegmentationLabelList, \n                                      classes=classes, src_img_size=train_img_dims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = item_list.add_test_folder(path_test, label=\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nbatch_size = 8\n\n# TODO add data augmentation\ntfms = ([], [])\n# tfms = get_transforms()\n\nitem_list = item_list.transform(tfms, tfm_y=True, size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (item_list\n        .databunch(bs=batch_size)\n        .normalize(imagenet_stats) # use same stats as pretrained model\n       )  \nassert data.test_ds is not None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(2, figsize=(15, 10), class_names=class_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_metric(pred, targs, threshold=0):\n    pred = (pred > threshold).float()\n    targs = targs.float()  # make sure target is float too\n    return 2.0 * (pred*targs).sum() / ((pred+targs).sum() + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metrics = [dice_metric]\n\ncallback_fns = [\n    # update a graph of learner stats and metrics after each epoch\n    ShowGraph,\n\n    # save model at every metric improvement\n    partial(SaveModelCallback, every='improvement', monitor='dice_metric', name=f\"{fmt_now()}_unet_resnet18_stage1_best\"),\n    \n    # stop training if metric no longer improve\n    partial(EarlyStoppingCallback, monitor='dice_metric', min_delta=0.01, patience=2),\n]\n\nlearn = unet_learner(data, models.resnet18, metrics=metrics, wd=1e-2, callback_fns=callback_fns)\nlearn.model_dir = \"/kaggle/working/\"  # point to writable directory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.loss_func","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(15, max_lr=1e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save(f\"{fmt_now()}_unet_resnet18_stage1\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lth {learn.model_dir}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### unfreeze and differential learing rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(15, max_lr=slice(5e-7, 5e-6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save(f\"{fmt_now()}_unet_resnet18_stage2\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Further tuning with larger images"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = (336, 528)\nitem_list = item_list.label_from_func(func=get_masks_rle, label_cls=MultiLabelSegmentationLabelList, \n                                      classes=classes, src_img_size=train_img_dims)\nitem_list = item_list.add_test_folder(path_test, label=\"\")\nbatch_size = 8\ntfms = ([], [])\nitem_list = item_list.transform(tfms, tfm_y=True, size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (item_list\n        .databunch(bs=batch_size)\n        .normalize(imagenet_stats) # use same stats as pretrained model\n       )  \nassert data.test_ds is not None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(2, figsize=(15, 10), class_names=class_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data = data\ndata.train_ds[0][0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 2e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(5, slice(lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr=5e-5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(5, slice(lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save(f\"{fmt_now()}_unet_resnet18_stage3\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.show_results(imgsize=8, class_names=class_names)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}