{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is just gpu version of my previous [notebook](https://www.kaggle.com/vineeth1999/hubmap-eda-pytorch-efficientunet-offline-training/notebook) Since EDA filled my gpu fully, I am running the training process seperately."},{"metadata":{},"cell_type":"markdown","source":"<img src='https://www.ml.cmu.edu/news/news-archive/2018/september/research-scientists-will-help-build-3d-cellular-map-of-human-body-machine-learning.jpg'>\n<h1><center>HuBMAP: Hacking the Kidney - Training and Inference</center><h1>"},{"metadata":{},"cell_type":"markdown","source":"# Pytorch Modelling GPU Offline"},{"metadata":{},"cell_type":"markdown","source":"We are using **pytorch** implementation of **UNet** Model implemented in **https://github.com/qubvel/segmentation_models.pytorch** and this is getting installed offline."},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/segmentationmodelspytorch/segmentation_models/efficientnet_pytorch-0.6.3.xyz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/segmentation-models-pytorch-0.1.2.xyz /tmp/pip/cache/segmentation_models_pytorch-0.1.2.tar.gz\n!cp ../input/segmentationmodelspytorch/segmentation_models/timm-0.1.20-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/segmentationmodelspytorch/segmentation_models/timm-0.2.1-py3-none-any.whl /tmp/pip/cache/\n!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch\n!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Necessary Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nimport torch\nfrom torch import nn\nimport torchvision\nimport cv2\nimport os\nimport numpy as np\nimport pandas as pd\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom scipy.ndimage.interpolation import zoom\nimport albumentations as A\nfrom torch.nn import functional as F\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport zipfile\nimport time\nimport random\nfrom albumentations.pytorch import ToTensorV2\nfrom segmentation_models_pytorch.unet import Unet\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b0-355c32eb.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b1-f1951068.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b2-8bb594d6.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b3-5fb5a3c3.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b4-6ed6700e.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b5-b6417697.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b6-c76e70fd.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/efficientnet-pytorch-b0-b7/efficientnet-b7-dcc49843.pth /root/.cache/torch/hub/checkpoints/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\nsz = 256  \nreduce = 4\nTH = 0.39 ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef mask2enc(mask, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs\n\n#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with bug fix\ndef rle_encode_less_memory(img):\n    #watch out for the bug\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class HuBMAPDataset(Dataset):\n    def __init__(self, ids, phase):\n        self.ids = ids\n        if phase=='train':\n            self.transform = get_train_transform()\n        else:\n            self.transform = get_val_transform()\n        \n    def __getitem__(self, idx):\n        name = self.ids[idx]\n        print(name)\n        img = cv2.imread(f\"../input/256256-hubmap/train/{name}\").astype(\"float32\")[:,:,::-1]\n        img /= 255.\n        mask = cv2.imread(f\"../input/256256-hubmap/masks/{name}\")[:,:,0:1]\n\n        transformed = self.transform(image=img, mask=mask)\n        img = transformed['image']\n        mask = transformed['mask']\n        img = img.transpose(2,0,1).astype('float32')\n        mask = mask.transpose(2,0,1).astype('float32')\n        return img, mask\n\n    def __len__(self):\n        return len(self.ids)\n\n        \ndef get_train_transform():\n    return A.Compose([\n        A.HorizontalFlip(),\n            A.OneOf([\n                A.RandomContrast(),\n                A.RandomGamma(),\n                A.RandomBrightness(),\n                ], p=0.3),\n            A.OneOf([\n                A.ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),\n                A.GridDistortion(),\n                A.OpticalDistortion(distort_limit=2, shift_limit=0.5),\n                ], p=0.3),\n            A.ShiftScaleRotate(p=0.2),\n            A.Resize(256,256,always_apply=True),\n    ],p=1.)\n\ndef get_val_transform():\n    return A.Compose([\n        A.Resize(256,256,always_apply=True),\n    ],p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DataLoader"},{"metadata":{"trusted":true},"cell_type":"code","source":"directory_list = os.listdir('../input/256256-hubmap/train')\ndir_df = pd.DataFrame(directory_list, columns=['Image_Paths'])\ndir_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_train_valid_dataloader(df, fold):\n    train_ids = df.loc[~df.Folds.isin(fold), \"Image_Paths\"].values\n    val_ids = df.loc[df.Folds.isin(fold), \"Image_Paths\"].values\n    train_ds = HuBMAPDataset(train_ids, \"train\")\n    val_ds = HuBMAPDataset(val_ids, \"val\")\n    train_loader = DataLoader(train_ds, batch_size=16, pin_memory=True, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_ds, batch_size=4, pin_memory=True, shuffle=False, num_workers=4)\n    return train_loader, val_loader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class HuBMAP(nn.Module):\n    def __init__(self):\n        super(HuBMAP, self).__init__()\n        self.cnn_model = Unet('efficientnet-b5', encoder_weights=\"imagenet\", classes=1, activation=None)\n        #self.cnn_model.decoder.blocks.append(self.cnn_model.decoder.blocks[-1])\n        #self.cnn_model.decoder.blocks[-2] = self.cnn_model.decoder.blocks[-3]\n    \n    def forward(self, imgs):\n        img_segs = self.cnn_model(imgs)\n        return img_segs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss Function"},{"metadata":{},"cell_type":"markdown","source":"<img src = 'https://wikimedia.org/api/rest_v1/media/math/render/svg/80f87a71d3a616a0939f5360cec24d702d2593a2'>"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DiceLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True):\n        super(DiceLoss, self).__init__()\n\n    def forward(self, inputs, targets, smooth=1):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = F.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).sum()                            \n        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)  \n        \n        return dice\n    \n    \n    \nclass DiceBCELoss(nn.Module):\n    # Formula Given above.\n    def __init__(self, weight=None, size_average=True):\n        super(DiceBCELoss, self).__init__()\n\n    def forward(self, inputs, targets, smooth=1):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = F.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        intersection = (inputs * targets).mean()                            \n        dice_loss = 1 - (2.*intersection + smooth)/(inputs.mean() + targets.mean() + smooth)  \n        BCE = F.binary_cross_entropy(inputs, targets, reduction='mean')\n        Dice_BCE = BCE + dice_loss\n        \n        return Dice_BCE.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def HuBMAPLoss(images, targets, model, device):\n    model.to(device)\n    images = images.to(device)\n    targets = targets.to(device)\n    outputs = model(images)\n    criterion = DiceBCELoss()\n    loss = criterion(outputs, targets)\n    return loss, outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(epoch, model, device, optimizer, scheduler, trainloader):\n    model.train()\n    t = time.time()\n    total_loss = 0\n    for step, (images, targets) in enumerate(trainloader):\n        loss, outputs = HuBMAPLoss(images, targets, model, device)\n        loss.backward()\n        if ((step+1)%4==0 or (step+1)==len(trainloader)):\n            optimizer.step()\n            scheduler.step()\n            optimizer.zero_grad()\n        loss = loss.detach().item()\n        total_loss += loss\n        if ((step+1)%10==0 or (step+1)==len(trainloader)):\n            print(\n                    f'epoch {epoch} train step {step+1}/{len(trainloader)}, ' + \\\n                    f'loss: {total_loss/len(trainloader):.4f}, ' + \\\n                    f'time: {(time.time() - t):.4f}', end= '\\r' if (step + 1) != len(trainloader) else '\\n'\n                )\n\n            \n        \ndef valid_one_epoch(epoch, model, device, optimizer, scheduler, validloader):\n    model.eval()\n    t = time.time()\n    total_loss = 0\n    for step, (images, targets) in enumerate(validloader):\n        loss, outputs = HuBMAPLoss(images, targets, model, device)\n        loss = loss.detach().item()\n        total_loss += loss\n        if ((step+1)%4==0 or (step+1)==len(validloader)):\n            scheduler.step(total_loss/len(validloader))\n        if ((step+1)%10==0 or (step+1)==len(validloader)):\n            print(\n                    f'epoch {epoch} valid step {step+1}/{len(validloader)}, ' + \\\n                    f'loss: {total_loss/len(validloader):.4f}, ' + \\\n                    f'time: {(time.time() - t):.4f}', end= '\\r' if (step + 1) != len(validloader) else '\\n'\n                )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating Folds Column"},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 5\ngkf = GroupKFold(FOLDS)\ndir_df['Folds'] = 0\nfor fold, (tr_idx, val_idx) in enumerate(gkf.split(dir_df, groups=dir_df[dir_df.columns[0]].values)):\n    dir_df.loc[val_idx, 'Folds'] = fold","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The Real Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, (tr_idx, val_idx) in enumerate(gkf.split(dir_df, groups=dir_df[dir_df.columns[0]].values)):\n    if fold>1:\n        break\n    trainloader, validloader = prepare_train_valid_dataloader(dir_df, [fold])\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = HuBMAP().to(device)\n    optimizer = Adam(model.parameters(), lr=5e-4)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1)\n    #num_epochs = 15\n    num_epochs = 2\n    for epoch in range(num_epochs):\n        train_one_epoch(epoch, model, device, optimizer, scheduler, trainloader)\n        with torch.no_grad():\n            valid_one_epoch(epoch, model, device, optimizer, scheduler, validloader)\n    torch.save(model.state_dict(),f'FOLD-{fold}-model.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}