{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi! I'm kaggle newbi!\n\nI use catalyst, smp, and albumentations.\nThey run on **pytorch** and are very convinience tools!\n\nSegmentation_models(smp) is High level API for image segmentation.\nhttps://github.com/qubvel/segmentation_models.pytorch\n\nIf you want to use other models or pretrained encoder, you just do type this.\n```\nENCODER = 'efficientnet-b3'\n```\n\nAlbumentations is a Python library for image augmentation.\nhttps://github.com/albumentations-team/albumentations\n\nIf you want to augmentate images, you just do type this!\n```\nalbu.Transpose(p=0.5), \nalbu.RandomRotate90(3)\n```\n\nThis notebook create on google colab. So If you load all data to google drive, you can run this notebook on colab!\n(Is there any method what I up load notebook with printed shell? I want to display result of this with code.)\n\n\nI challenge to submit test data, but my kaggle kernel shutdown when I convert mask to encoding.\nhaha..  :0\n\nAny comments are welcome! I really appreciate your opinion.\n"},{"metadata":{},"cell_type":"markdown","source":"# 1. Install library"},{"metadata":{"id":"SgdzO08_jSGC","trusted":false},"cell_type":"code","source":"from IPython.display import clear_output\n!pip install catalyst\n!pip install segmentation_models_pytorch\n!pip install albumentations==0.3.2\n!pip install -U git+https://github.com/albu/albumentations --no-cache-dir # for albu.lambda\nclear_output()","execution_count":null,"outputs":[]},{"metadata":{"id":"yymnjqf8N7_6","trusted":false},"cell_type":"code","source":"from PIL import Image\nimport tifffile as tiff\nimport subprocess\nimport pandas as pd\nfrom IPython.display import clear_output\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport cv2\nimport os\nfrom tqdm.notebook import tqdm\nimport zipfile\nfrom sklearn.model_selection import train_test_split\n\n\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport albumentations as albu\n#from albumentations import torch as AT\nfrom albumentations import Compose,Resize,OneOf,RandomBrightness,RandomContrast,Normalize,HorizontalFlip,Blur,ElasticTransform,GridDistortion,OpticalDistortion,GaussNoise \nfrom albumentations.pytorch import ToTensor\nfrom catalyst.data import Augmentor\nfrom catalyst.dl import utils\nfrom catalyst.data.reader import ScalarReader, ReaderCompose, LambdaReader#ImageReader\nfrom catalyst.dl.runner import SupervisedRunner\n#from catalyst.contrib.models.segmentation import Unet\nfrom catalyst.dl.callbacks import DiceCallback, EarlyStoppingCallback, InferCallback, CheckpointCallback\nfrom catalyst.dl.callbacks import JaccardCallback,PrecisionRecallF1ScoreCallback\nimport segmentation_models_pytorch as smp\n\n\nseed = 1015\nnp.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Create 256 * 256 images\n(Thanks Iafoss for your share!)\n\nI save files on google drive, run this part only one time.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_meta = pd.read_csv('/content/drive/MyDrive/kaggledrive/kidney/data/train.csv').set_index('id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load original data to google drive\n# load it to colab\n\ntrain_path = \"/content/drive/MyDrive/kaggledrive/kidney/data/train_img/\"\nfor idx in train_meta.index:\n    subprocess.call([\"cp\",\n                     f\"{train_path}{idx}.json\",\n                     f\"{idx}.json\"] ,shell = False)\n    subprocess.call([\"cp\",\n                     f\"{train_path}{idx}-anatomical-structure.json\",\n                     f\"{idx}-anatomical-structure.json\"] ,shell = False)\n    subprocess.call([\"cp\",\n                    f\"{train_path}{idx}.tiff\",\n                    f\"{idx}.tiff\"] ,shell = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions to convert encoding to mask and mask to encoding\ndef enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef mask2enc(mask, shape, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA = \"\"\nsz = 256   #the size of tiles\nreduce = 4 #reduce the original images by 4 times \nOUT_TRAIN = 'train_object.zip'\nOUT_MASKS = 'masks_object.zip'\n\ns_th = 40  #saturation blancking threshold\np_th = 200*sz//256 #threshold for the minimum number of pixels\n\nx_tot,x2_tot = [],[]\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out, zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for index, encs in tqdm(train_meta.iterrows(),total=len(train_meta)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n        if len(img.shape) == 5:img = np.transpose(img.squeeze(), (1,2,0))\n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n\n        #add padding to make the image dividable into tiles\n        shape = img.shape\n        pad0,pad1 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz), (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=0)\n        mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],constant_values=0)\n\n        #split image and mask into tiles using the reshape+transpose trick\n        img = cv2.resize(img,(img.shape[0]//reduce,img.shape[1]//reduce),interpolation = cv2.INTER_AREA)\n        \n        img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n        img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n        mask = cv2.resize(mask,(mask.shape[0]//reduce,mask.shape[1]//reduce),interpolation = cv2.INTER_NEAREST)\n        mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n        mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n\n        #write data\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue\n            if m.sum() <= 1 : continue\n    \n            x_tot.append((im/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((im/255.0)**2).reshape(-1,3).mean(0))\n            \n            im = cv2.imencode('.png',cv2.cvtColor(im, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n\n            m = cv2.imencode('.png',m)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n\n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save data to google drive\n\npath = \"/content/drive/MyDrive/kaggledrive/kidney/data/\"\n!cp train_object.zip \"{path}train_object_256.zip\"\n!cp masks_object.zip \"{path}masks_object_256.zip\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_meta = pd.read_csv('/content/drive/MyDrive/kaggledrive/kidney/data/train.csv').set_index('id')","execution_count":null,"outputs":[]},{"metadata":{"id":"lJxV0gCjb_Hj","trusted":false},"cell_type":"code","source":"# make directory on colab\n!mkdir /content/train_object_256\n!mkdir /content/masks_object_256\n# make logs directory on colab\n!mkdir logs \n\n# load 256 256 data\npath = \"/content/drive/MyDrive/kaggledrive/kidney/data/\"\n!cp \"{path}train_object_256.zip\" \"/content/train_object_256/train_object.zip\"\n!cp \"{path}masks_object_256.zip\" \"/content/masks_object_256/masks_object.zip \"\n\n!unzip \"/content/train_object_256/train_object.zip\"  -d \"/content/train_object_256\"\n!unzip \"/content/masks_object_256/masks_object.zip \" -d \"/content/masks_object_256\"\n\nclear_output()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Create Dataset"},{"metadata":{"id":"K5T-geYiki_5","trusted":false},"cell_type":"code","source":"namelist = []\nwith zipfile.ZipFile(\"/content/train_object_256/train_object.zip\", 'r') as img_arch:\n    namelist = img_arch.namelist().copy()","execution_count":null,"outputs":[]},{"metadata":{"id":"Axea1WADd8rD","trusted":false},"cell_type":"code","source":"class Kidney_Dataset(torch.utils.data.Dataset):\n    def __init__(self, namelist, \n                 transform=None,\n                 preprocessing=None,\n                 classes=1, \n                 augmentation=None, \n                ):\n        self.namelist = namelist\n        self.transforms = transform\n        self.classes = classes\n        self.preprocessing = preprocessing\n        self.augmentation = augmentation\n        self.imgsize = 256\n        '''\n        self.resize =  Compose([\n                                  albu.Resize(height = self.imgsize, width = self.imgsize),\n                               ])\n        '''\n        self.to_tensor = Compose([\n                                  albu.Lambda(image= to_tensor, mask=to_tensor),\n                               ])\n        \n        \n    def __getitem__(self,index):\n        img_name = self.namelist[index]\n        # Read Data----------------------------------------\n        img = np.array(Image.open(\"train_object_256/\"+img_name))/256\n        mask = np.array(Image.open(\"masks_object_256/\"+img_name))[...,np.newaxis]\n        \n        # Using background channel is usefull? ...> need to experient\n        \n        #background  = np.ones(mask.shape)\n        #background = background - mask\n        #mask = np.concatenate([mask,background], axis = 2)\n\n        #apply augmentation\n        if self.augmentation:\n            sample = self.augmentation(image=img, mask=mask)\n            img, mask = sample['image'], sample['mask']\n        \n        # apply preprocessing\n        if self.preprocessing:\n            sample = self.preprocessing(image=img, mask=mask)\n            img, mask = sample['image'], sample['mask']\n\n        # reshape for converting to tensor\n        sample = self.to_tensor(image=img, mask=mask)\n        img, mask = sample['image'], sample['mask']\n        \n        return img, mask\n    \n    def __len__(self):\n        return len(self.namelist)","execution_count":null,"outputs":[]},{"metadata":{"id":"I2l4zxZAjt2v","trusted":false},"cell_type":"code","source":"def get_training_augmentation():\n    transform = [\n        albu.Transpose(p=0.5),   # * 2\n        albu.RandomRotate90(3),  # * 4\n        #albu.Rotate(p=1),\n\n        albu.ShiftScaleRotate(p = 1),\n        albu.RandomSizedCrop(min_max_height=(196, 256), height = 256, width = 256, p = 1),#(128,256)\n        albu.GridDistortion(p = 0.5), # * 2\n\n        #albu.GridDropout(ratio= 0.4, mask_fill_value = 0, p = 1)\n        #albu.GridDropout(ratio= 0.4, mask_fill_value = None, p = 0.5, random_offset = True)\n\n        ]\n    return albu.Compose(transform)\n\n\ndef to_tensor(x, **kwargs):\n    return x.transpose(2, 0, 1).astype('float32')\n\ndef get_preprocessing(preprocessing_fn):\n    _transform = [\n        albu.Lambda(image = preprocessing_fn),\n    ]\n    return albu.Compose(_transform)","execution_count":null,"outputs":[]},{"metadata":{"id":"sfSGogVwenly","trusted":false},"cell_type":"code","source":"dataset = Kidney_Dataset(namelist, augmentation = get_training_augmentation(),  classes=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"FWF_49oQk4Xz","outputId":"b2a68b26-be0d-4d34-b2d3-e03047215cf9","trusted":false},"cell_type":"code","source":"img_id = 17\nplt.figure(figsize=(10,10))\nimg, mask = dataset[img_id]\nplt.imshow(img.transpose([1,2,0])) \nplt.imshow(mask[0,:,:], alpha=0.3) \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Create model"},{"metadata":{"id":"YYh_HSJQmwPv","outputId":"b4e8b9fc-70cb-4c2d-91bf-3e7b7f3d6cd9","trusted":false},"cell_type":"code","source":"ENCODER = 'efficientnet-b3'\nENCODER_WEIGHTS = 'imagenet'\nDEVICE = 'cuda'\n\nACTIVATION = 'sigmoid'\n\nmodel = smp.Unet(\n    encoder_name=ENCODER, \n    encoder_weights=ENCODER_WEIGHTS, \n    in_channels = 3,\n    classes=1, \n    activation = ACTIVATION\n)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"10hzdZJEnT2Y","trusted":false},"cell_type":"code","source":"train_names, test_names = train_test_split(namelist, test_size=0.10, random_state=1015)\n\n#preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)\ntrain_dataset = Kidney_Dataset(\n    train_names,\n    augmentation=get_training_augmentation(), \n    preprocessing=None, #get_preprocessing(preprocessing_fn),\n    classes=1,\n)\n\nvalid_dataset = Kidney_Dataset(\n    test_names,\n    augmentation = None,\n    preprocessing= None, #get_preprocessing(preprocessing_fn),\n    classes=1,\n)\n\nBATCH_SIZE = 32\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, persistent_workers = True)\nvalid_loader = DataLoader(valid_dataset,  batch_size=40, shuffle=False, num_workers=2)\nloaders = {\n    \"train\": train_loader,\n    \"valid\": valid_loader\n}","execution_count":null,"outputs":[]},{"metadata":{"id":"xgS70NYknxqT","trusted":false},"cell_type":"code","source":"num_epochs = 100\noptimizer = torch.optim.Adam([\n    {'params': model.decoder.parameters(), 'lr': 1e-2}, \n    {'params': model.encoder.parameters(), 'lr': 1e-3},  \n])\n\nscheduler = ReduceLROnPlateau(optimizer, factor=0.15, patience=4)\ncriterion = smp.utils.losses.DiceLoss(eps=1.)\nrunner = SupervisedRunner(device=device)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Train"},{"metadata":{"id":"yHaed4JOn0Qf","outputId":"c6e61bda-1417-4a5c-ccc8-42ac5e74c431","trusted":false},"cell_type":"code","source":"logdir = \"./logs\"\nrunner.train(\n    model=model,\n    criterion=criterion,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    loaders=loaders,\n    callbacks=[DiceCallback(threshold = 0.5),\n               JaccardCallback(),\n               EarlyStoppingCallback(patience=10, min_delta=0.001),\n               ],\n    logdir=logdir,\n    num_epochs=num_epochs,\n    verbose=True\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"8sK-RKVU5qeh","outputId":"72b61cb9-e089-4f01-f830-613864ef4183","trusted":false},"cell_type":"code","source":"%load_ext tensorboard","execution_count":null,"outputs":[]},{"metadata":{"id":"IqOAUhJr7cfM","outputId":"43c0b734-0966-4da3-af45-3de8ace4b7d7","trusted":false},"cell_type":"code","source":"%tensorboard --logdir {'logs'}","execution_count":null,"outputs":[]},{"metadata":{"id":"YUAh338IomLX","trusted":false},"cell_type":"code","source":"# save model on google drive\n\nPATH = \"/content/drive/MyDrive/kaggledrive/kidney/model/\"\n#torch.save(runner.model, PATH +\"efficientnet_b3.pt\")\ntorch.save(runner.model, \"efficientnet_b3.pt\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Visualization of model perpomence"},{"metadata":{"id":"niik8La6zJ8Z","trusted":false},"cell_type":"code","source":"# load model\n\nPATH = \"/content/drive/MyDrive/kaggledrive/kidney/model/\"\nmodel_new = torch.load(PATH +\"efficientnet_b3.pt\")\nmodel_new.eval()\n\nrunner = SupervisedRunner(device=device)\nrunner.model = model_new","execution_count":null,"outputs":[]},{"metadata":{"id":"LgRWP7ur89Q1"},"cell_type":"markdown","source":"## Train_Set"},{"metadata":{"id":"Cdr2K4Hf-HV6","outputId":"cf594aa1-3cd3-4380-9edb-3c2312d1a391","trusted":false},"cell_type":"code","source":"num = 3\nfor i, train_batch in enumerate(loaders['train']):\n    plt.figure(figsize = (12,4))\n    train_sample, train_mask = train_batch\n    train_out = runner.predict_batch({\"features\": train_sample.cuda()})['logits']\n    plt.subplot(1,3,1)\n    \n    plt.imshow(train_sample[num].permute(1,2,0).numpy()[:,:,0],cmap='bone') \n    plt.gca().set_title(\"Original\")\n\n       \n    plt.subplot(1,3,2)\n    plt.imshow(train_sample[num].permute(1,2,0).numpy()[:,:,0], cmap='bone') \n    plt.imshow(train_mask[num,0,:,:].cpu().numpy(),alpha=0.3) \n    plt.gca().set_title(\"Real Tissue\")\n\n    plt.subplot(1,3,3)\n    plt.imshow(train_sample[num].permute(1,2,0).numpy()[:,:,0], cmap='bone')\n    plt.imshow(train_out[num,0,:,:].cpu().numpy(),alpha = 0.3)\n    plt.gca().set_title(\"Predicted Tissue\")\n    \n    plt.show()\n    if i >10:\n        break","execution_count":null,"outputs":[]},{"metadata":{"id":"fwGxYY1a-LUn"},"cell_type":"markdown","source":"# validation set"},{"metadata":{"id":"22OXh57cq_IW","outputId":"723afff9-7cdc-4e61-be6b-0640f96c9931","trusted":false},"cell_type":"code","source":"for i, test_batch in enumerate(loaders['valid']):\n    test_sample, test_mask = test_batch\n    test_out = runner.predict_batch({\"features\": test_sample.cuda()})['logits']\n    for j in range(len(test_out)):\n        plt.figure(figsize = (12,4))\n        plt.subplot(1,3,1)\n        \n        plt.imshow(test_sample[j].permute(1,2,0).numpy()[:,:,0], cmap='bone') \n        plt.gca().set_title(\"Original\")\n        \n        plt.subplot(1,3,2)\n        plt.imshow(test_sample[j].permute(1,2,0).numpy()[:,:,0], cmap='bone')\n        plt.imshow(test_mask[j,0,:,:].cpu().numpy(), alpha = 0.3) \n        plt.gca().set_title(\"Real Tissue\")\n\n        \n        plt.subplot(1,3,3)\n        plt.imshow(test_sample[j].permute(1,2,0).numpy()[:,:,0],cmap='bone')\n        plt.imshow(test_out[j,0,:,:].cpu().numpy(),alpha = 0.3)\n        plt.gca().set_title(\"Predicted Tissue\")\n        plt.show()\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}