{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n    #    print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport warnings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import *\nfrom fastai.utils.mem import *\nfrom fastai.imports import *\nfrom fastai.callbacks.hooks import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Using devic', device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/understanding_cloud_organization/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(ROOT_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_CSV_PATH = os.path.join(ROOT_DIR, 'train.csv')\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['ImageId'] = train_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntrain_df['ClassId'] = train_df['Image_Label'].apply(lambda x: x.split('_')[1])\ntrain_df['hasMask'] = ~ train_df['EncodedPixels'].isna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_df.pivot(index='ImageId', columns='ClassId', values='EncodedPixels')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO remove use_partial_data()\nitem_list = (SegmentationItemList.\n             from_df(df=train.reset_index(), path=os.path.join(ROOT_DIR, 'train_images'), cols=\"ImageId\")\n             .use_partial_data(sample_pct=0.1)  # use only a subset of data to speedup tests\n             .split_by_rand_pct(0.2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiLabelImageSegment(ImageSegment): \n    \n    def reconstruct(self, t:Tensor): \n        return MultiClassImageSegment(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bce_logits_floatify(input, target, reduction='mean'):\n    return F.binary_cross_entropy_with_logits(input, target.float(), reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_to_mask(rle, shape):\n    mask_img = open_mask_rle(rle, shape)\n    mask = mask_img.px.permute(0, 2, 1)\n    return mask\n\nclass MultiLabelSegmentationLabelList(SegmentationLabelList):\n    \"\"\"Return a single image segment with all classes\"\"\"\n    # adapted from https://forums.fast.ai/t/how-to-load-multiple-classes-of-rle-strings-from-csv-severstal-steel-competition/51445/2\n    \n    def __init__(self, items:Iterator, src_img_size=None, classes:Collection=None, **kwargs):\n        super().__init__(items=items, classes=classes, **kwargs)\n        self.loss_func = bce_logits_floatify\n        self.src_img_size = src_img_size\n        # add attributes to copy by new() \n        self.copy_new += [\"src_img_size\"]\n    \n    def open(self, rles):        \n        # load mask at full resolution\n        masks = torch.zeros((len(self.classes), *self.src_img_size)) # shape CxHxW\n        for i, rle in enumerate(rles):\n            if isinstance(rle, str):  # filter out NaNs\n                masks[i] = rle_to_mask(rle, self.src_img_size)\n        return MultiLabelImageSegment(masks)\n    \n    def analyze_pred(self, pred, thresh:float=0.0):\n        # binarize masks\n        return (pred > thresh).float()\n    \n    def reconstruct(self, t:Tensor): \n        return MultiLabelImageSegment(t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masks_rle(img):\n    \"\"\"Get RLE-encoded masks for this image\"\"\"\n    img = img.split(\"/\")[-1]  # get filename only\n    return train.loc[img, class_names].to_list()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = (84*2, 132*2)\nimg_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_dims = (1400, 2100)\nclass_names = [\"Fish\", \"Flower\", \"Gravel\", \"Sugar\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = [0, 1, 2, 3] # no need for a \"void\" class: if a pixel isn't in any mask, it is not labelled\nitem_list = item_list.label_from_func(func=get_masks_rle, label_cls=MultiLabelSegmentationLabelList, \n                                      classes=classes, src_img_size=train_img_dims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"item_list = item_list.add_test_folder(os.path.join(ROOT_DIR, 'test_images'), label=\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 16\n\n# TODO add data augmentation\ntfms = ([], [])\n# tfms = get_transforms()\n\nitem_list = item_list.transform(tfms, tfm_y=True, size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (item_list\n        .databunch(bs=batch_size)\n        .normalize(imagenet_stats) # use same stats as pretrained model\n       )  \nassert data.test_ds is not None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_metric(pred, targs, threshold=0):\n    pred = (pred > threshold).float()\n    targs = targs.float()  # make sure target is float too\n    return 2.0 * (pred*targs).sum() / ((pred+targs).sum() + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fmt_now():\n    return datetime.today().strftime('%Y%m%d-%H%M%S')\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metrics = [dice_metric]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = unet_learner(data, models.resnet34, metrics=metrics, wd=1e-2)\nlearn.model_dir = \"/kaggle/working/\"  # point to writable directory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"critation = BCEWithLogitsFlat()\nlearn.loss = critation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr=1e-4\nprint(\"training_started\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()\nfree = gpu_mem_get_free_no_cache()\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10, max_lr=lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save(f\"_unet_resnet34_stage1\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"free = gpu_mem_get_free_no_cache()\nfree","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(10, slice(1e-5, 1e-4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save(f\"_unet_resnet34_stage2\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, _ = learn.get_preds(ds_type=DatasetType.Test, with_loss=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_pred_masks(preds, shape=(4, 350, 525)):\n    \"\"\"Resize predicted masks and return them as a generator\"\"\"\n    for p in range(preds.shape[0]):\n        mask = MultiLabelImageSegment(preds[p])\n        yield mask.resize(shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_masks = resize_pred_masks(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_fnames = [p.name for p in data.test_dl.items]\nlen(test_fnames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def write_submission_file(filename, test_fnames, preds, threshold=0):\n    with open(filename, mode='w') as f:\n        f.write(\"Image_Label,EncodedPixels\\n\")\n\n        for img_name, masks in zip(tqdm(test_fnames), resize_pred_masks(preds)):\n            binary_masks = masks.px > threshold # TODO use activation instead\n            \n            for class_idx, class_name in enumerate(class_names):\n                rle = rle_encode(binary_masks[class_idx].numpy().T)\n                f.write(f\"{img_name}_{class_name},{rle}\\n\")\n\n    print(f\"Wrote '{f.name}'.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_file = f\"{fmt_now()}_submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_submission_file(submission_file, test_fnames, preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"free = gpu_mem_get_free_no_cache()\nfree","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}