{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# download kaggle dependent labraries \n# in kaggle  need to -> Dataset  \n!pip install pretrainedmodels\n!pip install segmentation_models_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport pathlib, sys, os, random, time\nimport numba, cv2, gc\nimport glob\n\nfrom sklearn.model_selection import KFold\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom tqdm.notebook import tqdm\nimport albumentations as A\nimport rasterio\nfrom rasterio.windows import Window\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as D\n\nimport torchvision\nfrom torchvision import transforms as T\n\n# set up seed for reproduction\ndef set_seeds(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nset_seeds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nDATA_PATH = '../input/hubmap-kidney-segmentation/'\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' \n\nimport logging\nlogging.basicConfig(filename='log.log',\n                    format='%(asctime)s - %(name)s - %(levelname)s -%(module)s:  %(message)s',\n                    datefmt='%Y-%m-%d %H:%M:%S ',\n                    level=logging.INFO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# used for converting the decoded image to rle mask\ndef rle_encode(im):\n    '''\n    im: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = im.flatten(order = 'F')\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape, order='F')\n\n# accelerate\n@numba.njit()\ndef rle_numba(pixels):\n    size = len(pixels)\n    points = []\n    if pixels[0] == 1: points.append(0)\n    flag = True\n    for i in range(1, size):\n        if pixels[i] != pixels[i-1]:\n            if flag:\n                points.append(i+1)\n                flag = False\n            else:\n                points.append(i+1 - points[-1])\n                flag = True\n    if pixels[-1] == 1: points.append(size-points[-1]+1)    \n    return points\n\ndef rle_numba_encode(image):\n    pixels = image.flatten(order = 'F')\n    points = rle_numba(pixels)\n    return ' '.join(str(x) for x in points)\n\n# crop the raw image.tiff\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate the dataset from the run-length encoded mask and tiff -> label y and image x \nidentity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n\nclass HubDataset(D.Dataset):\n\n    def __init__(self, path, tiff_ids, transform,\n                 window=256, overlap=32, threshold = 100, isvalid=False):\n        self.path = pathlib.Path(path)\n        self.tiff_ids = tiff_ids\n        self.overlap = overlap\n        self.window = window\n        self.transform = transform\n        self.csv = pd.read_csv((self.path / 'train.csv').as_posix(),\n                               index_col=[0])\n        self.threshold = threshold\n        self.isvalid = isvalid\n        \n        self.x, self.y, self.id = [], [], []\n        self.build_slices()\n        self.len = len(self.x)\n        self.as_tensor = T.Compose([\n            T.ToTensor(),\n            T.Normalize([0.625, 0.448, 0.688],\n                        [0.131, 0.177, 0.101]),\n        ])\n        \n    \n    def build_slices(self):\n        self.masks = []\n        self.files = []\n        self.slices = []\n        for i, filename in enumerate(self.csv.index.values):\n            if not filename in self.tiff_ids:\n                continue\n            \n            filepath = (self.path /'train'/(filename+'.tiff')).as_posix()\n            self.files.append(filepath)\n            \n            print('Transform', filename)\n            with rasterio.open(filepath, transform = identity) as dataset:\n                self.masks.append(rle_decode(self.csv.loc[filename, 'encoding'], dataset.shape))\n                slices = make_grid(dataset.shape, window=self.window, min_overlap=self.overlap)\n                \n                for slc in slices:\n                    x1,x2,y1,y2 = slc\n                    # print(slc)\n                    image = dataset.read([1,2,3],\n                            window=Window.from_slices((x1,x2),(y1,y2)))\n                    image = np.moveaxis(image, 0, -1)\n                    \n                    image = cv2.resize(image, (256, 256))\n                    masks = cv2.resize(self.masks[-1][x1:x2,y1:y2], (256, 256))\n                    \n                    if self.isvalid:\n                        self.slices.append([i,x1,x2,y1,y2])\n                        self.x.append(image)\n                        self.y.append(masks)\n                        self.id.append(filename)\n                    else:\n                        if self.masks[-1][x1:x2,y1:y2].sum() >= self.threshold or (image>32).mean() > 0.25:\n                            self.slices.append([i,x1,x2,y1,y2])\n                            \n                            self.x.append(image)\n                            self.y.append(masks)\n                            self.id.append(filename)\n    \n    # get data operation\n    def __getitem__(self, index):\n        image, mask = self.x[index], self.y[index]\n        augments = self.transform(image=image, mask=mask)\n        return self.as_tensor(augments['image']), augments['mask'][None]\n    \n    def __len__(self):\n        \"\"\"\n        Total number of samples in the dataset\n        \"\"\"\n        return self.len\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SoftDiceLoss(nn.Module):\n    def __init__(self, smooth=1., dims=(-2,-1)):\n        super(SoftDiceLoss, self).__init__()\n        self.smooth = smooth\n        self.dims = dims\n    \n    def forward(self, x, y):\n        tp = (x * y).sum(self.dims)\n        fp = (x * (1 - y)).sum(self.dims)\n        fn = ((1 - x) * y).sum(self.dims)\n        dc = (2 * tp + self.smooth) / (2 * tp + fp + fn + self.smooth)\n        dc = dc.mean()\n        \n        return 1 - dc\n\nbce_fn = nn.BCEWithLogitsLoss()\n# bce_fn = nn.BCELoss()\ndice_fn = SoftDiceLoss()\n    \ndef loss_fn(y_pred, y_true, ratio=1.0, hard=False):\n    bce = bce_fn(y_pred, y_true)\n    if hard:\n        dice = dice_fn((y_pred.sigmoid()).float() > 0.5, y_true)\n    else:\n        dice = dice_fn(y_pred.sigmoid(), y_true)\n    return ratio*bce + (1-ratio)*dice\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model train function \ndef train(model, train_loader, criterion, optimizer):\n    losses = []\n    for i, (image, target) in enumerate(train_loader):\n        image, target = image.to(DEVICE), target.float().to(DEVICE)\n        optimizer.zero_grad()\n        \n        output = model(image)\n        loss = criterion(output, target, 1, False)\n        loss.backward()\n        optimizer.step()\n        losses.append(loss.item())\n        # print('train, ', loss.item())\n    return np.array(losses).mean()\n\n# # make up all the patch to eval dice \ndef np_dice_score(probability, mask):\n    p = probability.reshape(-1)\n    t = mask.reshape(-1)\n\n    p = p>0.4\n    t = t>0.5\n    uion = p.sum() + t.sum()\n    \n    overlap = (p*t).sum()\n    dice = 2*overlap/(uion+0.001)\n    return dice\n\n# make up all the patch to eval dice with searching for the best threshold\n# def np_dice_score(probability, mask):\n#     threshold = 0\n#     dice_best = 0\n#     for nt in np.linspace(0.2, 0,8, 17):\n#         p = probability.reshape(-1)\n#         t = mask.reshape(-1)\n#         p = p > nt\n#         t = t > 0.5\n#         uion = p.sum() + t.sum()\n    \n#         overlap = (p*t).sum()\n#         dice = 2*overlap/(uion+0.001)\n#         if(dice > dice_best):\n#             threshold = nt\n#             dice_best = dice\n#         #print(nt, dice)\n\n#     return threshold, dice_best\n        \n# model eval function \ndef validation(model, val_loader, criterion):\n    val_probability, val_mask = [], []\n    model.eval()\n    with torch.no_grad():\n        for image, target in val_loader:\n            image, target = image.to(DEVICE), target.float().to(DEVICE)\n            output = model(image)\n            \n            output_ny = output.sigmoid().data.cpu().numpy()\n            target_np = target.data.cpu().numpy()\n            \n            val_probability.append(output_ny)\n            val_mask.append(target_np)\n            \n    val_probability = np.concatenate(val_probability)\n    val_mask = np.concatenate(val_mask)\n    \n    return np_dice_score(val_probability, val_mask)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHES = 8\nBATCH_SIZE = 4\n\nWINDOW=1024\nMIN_OVERLAP=40\nNEW_SIZE=320\n\ntrain_trfm = A.Compose([\n    # A.RandomCrop(NEW_SIZE*3, NEW_SIZE*3),\n    A.Resize(NEW_SIZE, NEW_SIZE),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomRotate90(),\n    A.OneOf([\n        A.HueSaturationValue(10,15,10),\n        A.CLAHE(clip_limit=2),\n        A.RandomBrightnessContrast(),            \n    ], p=0.3),\n    A.OneOf([\n        A.OpticalDistortion(p=0.3),\n        A.GridDistortion(p=0.1),\n        A.IAAPiecewiseAffine(p=0.3),\n    ], p=0.3),\n     A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=15, p=0.9, \n                                 border_mode=cv2.BORDER_REFLECT),\n])\n\nval_trfm = A.Compose([\n    # A.CenterCrop(NEW_SIZE, NEW_SIZE),\n    A.Resize(NEW_SIZE,NEW_SIZE),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomRotate90(),\n#     A.OneOf([\n#         A.RandomContrast(),\n#         A.RandomGamma(),\n#         A.RandomBrightness(),\n#         A.ColorJitter(brightness=0.07, contrast=0.07,\n#                    saturation=0.1, hue=0.1, always_apply=False, p=0.3),\n#         ], p=0.3),\n#     A.OneOf([\n#         A.OpticalDistortion(p=0.5),\n#         A.GridDistortion(p=0.5),\n#         A.IAAPiecewiseAffine(p=0.5),\n#     ], p=0.3),\n#     A.ShiftScaleRotate(),\n])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define model \n# model = get_model()\n# model = Unet(encoder_name=\"resnet34\",classes=1,activation=None)\nimport segmentation_models_pytorch as smp\n\nmodel = smp.Unet(\n    encoder_name=\"efficientnet-b5\",        # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n    encoder_weights=\"imagenet\",     # use `imagenet` pretreined weights for encoder initialization\n    in_channels=3,                  # model input channels (1 for grayscale images, 3 for RGB, etc.)\n    classes=1,                      # model output channels (number of classes in your dataset)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # csv write fast for public/A leaderboard 8 fold \n# # to be confirmed \n# trfm = T.Compose([\n#     T.ToPILImage(),\n#     T.Resize(NEW_SIZE),\n#     T.ToTensor(),\n#     T.Normalize([0.625, 0.448, 0.688],\n#                 [0.131, 0.177, 0.101]),\n# ])\n\n\n# #mold_path = '../input/trained_model/'\n# mold_path = '../input/dataweights/unet_8_fold/'\n# fold_models = []\n# for fold_model_path in glob.glob(mold_path + '*.pth'):\n#     fold_models.append(torch.load(fold_model_path))\n# print(len(fold_models))\n\n\n# p = pathlib.Path(DATA_PATH)\n\n# subm = {}\n\n\n# for i, filename in enumerate(p.glob('test/*.tiff')):\n#     print(f'{i+1} Predicting {filename.stem}')\n#     start_time = time.time()\n#     dataset = rasterio.open(filename.as_posix(), transform = identity)\n#     slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n#     preds = np.zeros(dataset.shape, dtype=np.uint8)\n#     print(slices.shape)\n    \n#     for (x1,x2,y1,y2) in slices:\n        \n#         image = dataset.read([1,2,3],\n#                     window=Window.from_slices((x1,x2),(y1,y2)))\n#         image = np.moveaxis(image, 0, -1)\n#         image = trfm(image)\n        \n        \n#         pred = None\n        \n        \n#         for fold_model in fold_models:\n            \n#             model.load_state_dict(fold_model)\n#             model.eval()\n#             model.to(DEVICE)\n            \n#             # with 3 times testifid\n#             with torch.no_grad():\n#                 image = image.to(DEVICE)\n#                 image = image.reshape(1, 3, 320, 320)\n                \n#                 score = model(image)[0][0]\n                            \n#                 score2 = model(torch.flip(image, [0, 3]))\n#                 score2 = torch.flip(score2, [3, 0])[0][0]\n\n#                 score3 = model(torch.flip(image, [1, 2]))\n#                 score3 = torch.flip(score3, [2, 1])[0][0]\n                \n#                 if pred is None:\n#                     pred = (score + score2 + score3) / 3.0\n#                 else:\n#                     pred += (score + score2 + score3) / 3.0\n# #                 if pred is None:\n# #                     pred = score\n# #                 else:\n# #                     pred += score\n        \n#         pred = pred / len(fold_models)\n#         score_sigmoid = pred.sigmoid().cpu().numpy()\n#         score_sigmoid = cv2.resize(score_sigmoid, (WINDOW, WINDOW))\n        \n        \n#         preds[x1:x2,y1:y2] = (score_sigmoid > 0.4).astype(np.uint8)\n        \n        \n#     subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds)}\n       \n#     print((time.time()-start_time)/60**1) \n#     del preds\n#     gc.collect();\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# csv write fast for public/A leaderboard single fold \n# to be confirmed \ntrfm = T.Compose([\n    T.ToPILImage(),\n    T.Resize(NEW_SIZE),\n    T.ToTensor(),\n    T.Normalize([0.625, 0.448, 0.688],\n                [0.131, 0.177, 0.101]),\n])\n\n# define your model path \nmold_path = '../input/trained_model/'\n\nmodel.load_state_dict(mold_path)\nmodel.eval()\nmodel.to(DEVICE)\n            \n\np = pathlib.Path(DATA_PATH)\n\nsubm = {}\n\n\nfor i, filename in enumerate(p.glob('test/*.tiff')):\n    print(f'{i+1} Predicting {filename.stem}')\n    start_time = time.time()\n    dataset = rasterio.open(filename.as_posix(), transform = identity)\n    slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n    preds = np.zeros(dataset.shape, dtype=np.uint8)\n    print(slices.shape)\n    \n    for (x1,x2,y1,y2) in slices:\n        \n        image = dataset.read([1,2,3],\n                    window=Window.from_slices((x1,x2),(y1,y2)))\n        image = np.moveaxis(image, 0, -1)\n        image = trfm(image)\n        \n        \n        pred = None\n        \n        \n        \n            \n            \n        # with 3 times testifid\n        with torch.no_grad():\n            image = image.to(DEVICE)\n            image = image.reshape(1, 3, 320, 320)\n\n            score = model(image)[0][0]\n\n            score2 = model(torch.flip(image, [0, 3]))\n            score2 = torch.flip(score2, [3, 0])[0][0]\n\n            score3 = model(torch.flip(image, [1, 2]))\n            score3 = torch.flip(score3, [2, 1])[0][0]\n\n            if pred is None:\n                pred = (score + score2 + score3) / 3.0\n            else:\n                pred += (score + score2 + score3) / 3.0\n#                 if pred is None:\n#                     pred = score\n#                 else:\n#                     pred += score\n        \n        pred = pred / len(fold_models)\n        score_sigmoid = pred.sigmoid().cpu().numpy()\n        score_sigmoid = cv2.resize(score_sigmoid, (WINDOW, WINDOW))\n        \n        \n        preds[x1:x2,y1:y2] = (score_sigmoid > 0.4).astype(np.uint8)\n        \n        \n    subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds)}\n       \n    print((time.time()-start_time)/60**1) \n    del preds\n    gc.collect();\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict(subm, orient='index')\nsubmission.to_csv('./submission_unet_8_fold.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}