{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a perfect notebook for segmentation from scratch. I put so many things from what I have leanred. In this notebook I used Encoded pixels for segmentation. A lot of segmentation dataset includes masks that are encoded.So you have to decode it and build a model on it. Lets start.\n\nIf you find this notebook is helpful to you, pls upvote.\n\n","metadata":{}},{"cell_type":"markdown","source":"# Install and Import libraries","metadata":{}},{"cell_type":"code","source":"!pip install catalyst==20.12\n!pip install pretrainedmodels\n!pip install git+https://github.com/qubvel/segmentation_models.pytorch\n!pip install pytorch_toolbelt\n!pip install torchvision","metadata":{"execution":{"iopub.status.busy":"2022-10-01T09:56:40.096631Z","iopub.execute_input":"2022-10-01T09:56:40.097171Z","iopub.status.idle":"2022-10-01T09:58:12.525835Z","shell.execute_reply.started":"2022-10-01T09:56:40.097059Z","shell.execute_reply":"2022-10-01T09:58:12.524265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport tqdm\nimport time\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torchvision\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\nimport torchvision.transforms.functional as TF\nfrom torch.optim import lr_scheduler\nimport torchvision.transforms as transforms  \n\nfrom sklearn.model_selection import train_test_split\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torch.utils.tensorboard import SummaryWriter\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-01T09:58:12.529123Z","iopub.execute_input":"2022-10-01T09:58:12.530128Z","iopub.status.idle":"2022-10-01T09:58:17.456370Z","shell.execute_reply.started":"2022-10-01T09:58:12.530045Z","shell.execute_reply":"2022-10-01T09:58:17.455043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"#unzip files\nimport zipfile\ndirs = ['train.zip','test.zip', 'train_masks.zip', 'train_masks.csv.zip']\nfor x in dirs:\n    with zipfile.ZipFile('../input/carvana-image-masking-challenge/' + x, 'r') as z:\n        z.extractall()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T09:58:17.461780Z","iopub.execute_input":"2022-10-01T09:58:17.465639Z","iopub.status.idle":"2022-10-01T10:02:48.936404Z","shell.execute_reply.started":"2022-10-01T09:58:17.465596Z","shell.execute_reply":"2022-10-01T10:02:48.935087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dataframe analysis\ndf = pd.read_csv('./train_masks.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:48.939806Z","iopub.execute_input":"2022-10-01T10:02:48.940112Z","iopub.status.idle":"2022-10-01T10:02:49.422678Z","shell.execute_reply.started":"2022-10-01T10:02:48.940083Z","shell.execute_reply":"2022-10-01T10:02:49.421188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df shape(rows x columns)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:49.424829Z","iopub.execute_input":"2022-10-01T10:02:49.425674Z","iopub.status.idle":"2022-10-01T10:02:49.436375Z","shell.execute_reply.started":"2022-10-01T10:02:49.425630Z","shell.execute_reply":"2022-10-01T10:02:49.433884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualization on dataframe\ng = sns.countplot(df['img'].apply(lambda x: x.split('_')[1]))\ndf['img'].apply(lambda x: x.split('_')[1]).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:49.439677Z","iopub.execute_input":"2022-10-01T10:02:49.443405Z","iopub.status.idle":"2022-10-01T10:02:49.862357Z","shell.execute_reply.started":"2022-10-01T10:02:49.443359Z","shell.execute_reply":"2022-10-01T10:02:49.860341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking null values on encoded pixels\ndf['rle_mask'].isnull().any()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:49.864347Z","iopub.execute_input":"2022-10-01T10:02:49.866563Z","iopub.status.idle":"2022-10-01T10:02:49.878180Z","shell.execute_reply.started":"2022-10-01T10:02:49.866518Z","shell.execute_reply":"2022-10-01T10:02:49.876530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't have null values so we are fine to continute","metadata":{}},{"cell_type":"code","source":"#getting image from train dataset\ndef get_image(x, folder: str='train'):\n    data_folder = f'{folder}'\n    image_path = os.path.join(data_folder, x)\n    img = cv2.imread(image_path)\n    img = cv2. cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:49.880483Z","iopub.execute_input":"2022-10-01T10:02:49.881258Z","iopub.status.idle":"2022-10-01T10:02:49.890266Z","shell.execute_reply.started":"2022-10-01T10:02:49.881200Z","shell.execute_reply":"2022-10-01T10:02:49.888547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = 'd0392fd5feb6_03.jpg'\nplt.imshow(get_image(x))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:49.892338Z","iopub.execute_input":"2022-10-01T10:02:49.892874Z","iopub.status.idle":"2022-10-01T10:02:50.820835Z","shell.execute_reply.started":"2022-10-01T10:02:49.892833Z","shell.execute_reply":"2022-10-01T10:02:50.819473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(get_image(x).shape) #(height, width, channels)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:50.843097Z","iopub.execute_input":"2022-10-01T10:02:50.844324Z","iopub.status.idle":"2022-10-01T10:02:50.993529Z","shell.execute_reply.started":"2022-10-01T10:02:50.844260Z","shell.execute_reply":"2022-10-01T10:02:50.991623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#decoding on encoded pixels \ndef rle_decode(mask_rle: str = '', shape: tuple = (1280, 1918)):\n    '''\n    Decode rle encoded mask.\n    \n    :param mask_rle: run-length as string formatted (start length)\n    :param shape: (height, width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:51.001765Z","iopub.execute_input":"2022-10-01T10:02:51.006139Z","iopub.status.idle":"2022-10-01T10:02:51.027832Z","shell.execute_reply.started":"2022-10-01T10:02:51.006066Z","shell.execute_reply":"2022-10-01T10:02:51.026166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualization on decoded masks\nmask_rle = df['rle_mask'][0]\nmask = rle_decode(mask_rle)\nplt.imshow(mask)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:51.034758Z","iopub.execute_input":"2022-10-01T10:02:51.035227Z","iopub.status.idle":"2022-10-01T10:02:52.340426Z","shell.execute_reply.started":"2022-10-01T10:02:51.035167Z","shell.execute_reply":"2022-10-01T10:02:52.338621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize train images and train masks together","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(25, 16))\nfor j, im_id in enumerate(np.random.choice(df['img'].unique(), 4)):\n    for i, (idx, row) in enumerate(df.loc[df['img'] == im_id].iterrows()):\n        ax = fig.add_subplot(5, 4, j*4+i+1, xticks=[], yticks=[])\n        im = Image.open(f\"./train/{row['img']}\")\n        plt.imshow(im)\n        mask_rle = row['rle_mask']\n        try:\n            mask = rle_decode(mask_rle)\n        except:\n            mask = np.zeros((1280, 1918))\n        plt.imshow(mask, alpha=0.5, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:52.347574Z","iopub.execute_input":"2022-10-01T10:02:52.351835Z","iopub.status.idle":"2022-10-01T10:02:55.928507Z","shell.execute_reply.started":"2022-10-01T10:02:52.351758Z","shell.execute_reply":"2022-10-01T10:02:55.927103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare for training","metadata":{}},{"cell_type":"code","source":"n_train = len(os.listdir('./train'))\nn_test = len(os.listdir('./test'))\nprint(f\"There are {n_train} train images.\")\nprint(f\"There are {n_test} test images.\")","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:55.930457Z","iopub.execute_input":"2022-10-01T10:02:55.931205Z","iopub.status.idle":"2022-10-01T10:02:56.064549Z","shell.execute_reply.started":"2022-10-01T10:02:55.931162Z","shell.execute_reply":"2022-10-01T10:02:56.063158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_mask_count = df.loc[df['rle_mask'].isnull() == False, 'img'].value_counts().\\\nreset_index().rename(columns={'index': 'img_id', 'img': 'count'})\ntrain_ids, valid_ids = train_test_split(id_mask_count['img_id'].values, random_state=42, stratify=id_mask_count['count'], test_size=0.1)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.066671Z","iopub.execute_input":"2022-10-01T10:02:56.067145Z","iopub.status.idle":"2022-10-01T10:02:56.089492Z","shell.execute_reply.started":"2022-10-01T10:02:56.067092Z","shell.execute_reply":"2022-10-01T10:02:56.088289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class Carvana(Dataset):\n    def __init__(self, df: pd.DataFrame = None, datatype: str = 'train', img_ids: np.array=None, transforms = A.Compose([A.HorizontalFlip(),ToTensorV2()]), preprocessing=None):\n        self.df = df\n        if datatype != 'test':\n            self.data_folder = './train'\n        else:\n            self.data_folder = './test'\n        self.img_ids = img_ids\n        self.transforms = transforms\n    def __len__(self):\n        return len(self.img_ids)\n        \n    def __getitem__(self, index):\n        image_name = self.img_ids[index]\n        mask = make_mask(self.df, image_name)\n        image_path = os.path.join(self.data_folder, image_name)\n        img = cv2.imread(image_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        if self.transforms is not None:\n            augmentations = self.transforms(image=img, mask=mask)\n            img = augmentations[\"image\"]\n            mask = augmentations[\"mask\"]\n        return img, mask\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.092899Z","iopub.execute_input":"2022-10-01T10:02:56.093216Z","iopub.status.idle":"2022-10-01T10:02:56.104592Z","shell.execute_reply.started":"2022-10-01T10:02:56.093189Z","shell.execute_reply":"2022-10-01T10:02:56.102860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_mask(df: pd.DataFrame, image_name: str='img.jpg', shape: tuple=(1280, 1918)):\n    encoded_masks = df.loc[df['img'] == image_name, 'rle_mask']\n    masks = np.zeros((shape[0], shape[1], 1), dtype=np.float32)\n    \n    for idx, label in enumerate(encoded_masks.values):\n        mask = rle_decode(label)\n        masks[:,:,idx]=mask\n    return masks","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.106404Z","iopub.execute_input":"2022-10-01T10:02:56.107267Z","iopub.status.idle":"2022-10-01T10:02:56.120554Z","shell.execute_reply.started":"2022-10-01T10:02:56.107223Z","shell.execute_reply":"2022-10-01T10:02:56.119234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Albumentation and data loader","metadata":{}},{"cell_type":"code","source":"train_transform = A.Compose([\n    A.Resize(320, 320),\n    A.Rotate(limit=35,p=1.0),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.1),\n    A.Normalize(\n        mean=[0.0,0.0,0.0],\n        std = [1.0,1.0,1.0],\n        max_pixel_value=255.0\n    ),\n    ToTensorV2(transpose_mask=True)  \n])\nvalidation_transform = A.Compose([\n    A.Resize(320, 320),\n    A.Normalize(\n        mean = [0.0,0.0,0.0],\n        std = [1.0,1.0,1.0],\n        max_pixel_value=255.0,\n    ),\n    ToTensorV2(transpose_mask=True)\n])","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.122793Z","iopub.execute_input":"2022-10-01T10:02:56.123781Z","iopub.status.idle":"2022-10-01T10:02:56.135042Z","shell.execute_reply.started":"2022-10-01T10:02:56.123740Z","shell.execute_reply":"2022-10-01T10:02:56.133742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir = \"./train\"\nmask_dir = \"./train_masks\"\nnum_workers = 0\nbs = 10\ntrain_dataset = Carvana(df=df, datatype='train', img_ids=train_ids, transforms=train_transform)\nvalid_dataset = Carvana(df=df, datatype='valid', img_ids=valid_ids, transforms=validation_transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers)\nvalid_loader = DataLoader(valid_dataset, batch_size=bs, shuffle=False, num_workers=num_workers)\nloaders = {\"train\": train_loader, \"valid\": valid_loader}","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.137134Z","iopub.execute_input":"2022-10-01T10:02:56.137590Z","iopub.status.idle":"2022-10-01T10:02:56.151640Z","shell.execute_reply.started":"2022-10-01T10:02:56.137532Z","shell.execute_reply":"2022-10-01T10:02:56.150352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize transformed images","metadata":{}},{"cell_type":"code","source":"def show_transform_images(dataloader):\n    batch = next(iter(dataloader))\n    images, labels = batch\n    \n    grid = torchvision.utils.make_grid(images, nrow=3)\n    print(grid.size())\n    plt.figure(figsize=(100,100))\n    #plt.imshow(grid.permute(1, 2, 0))\n    #plt.figure(figsize=(11,11))\n    plt.imshow(np.transpose(grid,(1,2,0)))","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.153578Z","iopub.execute_input":"2022-10-01T10:02:56.154572Z","iopub.status.idle":"2022-10-01T10:02:56.167909Z","shell.execute_reply.started":"2022-10-01T10:02:56.154503Z","shell.execute_reply":"2022-10-01T10:02:56.166717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_transform_images(train_loader)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:02:56.170137Z","iopub.execute_input":"2022-10-01T10:02:56.171194Z","iopub.status.idle":"2022-10-01T10:03:04.037171Z","shell.execute_reply.started":"2022-10-01T10:02:56.170916Z","shell.execute_reply":"2022-10-01T10:03:04.036015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# U net pretrained model","metadata":{}},{"cell_type":"code","source":"import segmentation_models_pytorch as smp\nENCODER = 'resnet50'\nENCODER_WEIGHTS = 'imagenet'\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nACTIVATION = None\nmodel = smp.Unet(\n    encoder_name=ENCODER, \n    encoder_weights=ENCODER_WEIGHTS, \n    classes=1, \n    activation=ACTIVATION,\n)\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:03:04.038767Z","iopub.execute_input":"2022-10-01T10:03:04.039740Z","iopub.status.idle":"2022-10-01T10:03:16.226170Z","shell.execute_reply.started":"2022-10-01T10:03:04.039699Z","shell.execute_reply":"2022-10-01T10:03:16.224649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting parameters ","metadata":{}},{"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom catalyst.dl.runner import SupervisedRunner\nnum_epochs = 19\nlogdir = \"./logs/segmentation\"\n\n# model, criterion, optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-2)\nscheduler = ReduceLROnPlateau(optimizer, factor=0.15, patience=2)\ncriterion = smp.losses.DiceLoss(mode='multilabel')\nrunner = SupervisedRunner()","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:03:16.228674Z","iopub.execute_input":"2022-10-01T10:03:16.229325Z","iopub.status.idle":"2022-10-01T10:03:17.112391Z","shell.execute_reply.started":"2022-10-01T10:03:16.229268Z","shell.execute_reply":"2022-10-01T10:03:17.111030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training ","metadata":{}},{"cell_type":"code","source":"from catalyst.dl.callbacks import DiceCallback, EarlyStoppingCallback, InferCallback, CheckpointCallback\nrunner.train(\n    model=model,\n    criterion=criterion,\n    optimizer=optimizer,\n    loaders=loaders,\n    scheduler=ReduceLROnPlateau(optimizer, factor=0.15, patience=2),\n    callbacks=[DiceCallback(), EarlyStoppingCallback(patience=5, min_delta=0.001)],\n    logdir = logdir,\n    num_epochs = num_epochs,\n    valid_loader=\"valid\",\n    verbose=True,\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T10:03:17.114300Z","iopub.execute_input":"2022-10-01T10:03:17.114701Z","iopub.status.idle":"2022-10-01T11:16:53.970757Z","shell.execute_reply.started":"2022-10-01T10:03:17.114657Z","shell.execute_reply":"2022-10-01T11:16:53.969289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorboard","metadata":{}},{"cell_type":"code","source":"# Load the TensorBoard notebook extension\n%load_ext tensorboard","metadata":{"execution":{"iopub.status.busy":"2022-10-01T11:16:53.972572Z","iopub.execute_input":"2022-10-01T11:16:53.974803Z","iopub.status.idle":"2022-10-01T11:16:54.001935Z","shell.execute_reply.started":"2022-10-01T11:16:53.974761Z","shell.execute_reply":"2022-10-01T11:16:54.000747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%tensorboard --logdir runs","metadata":{"execution":{"iopub.status.busy":"2022-10-01T11:16:54.005624Z","iopub.execute_input":"2022-10-01T11:16:54.006039Z","iopub.status.idle":"2022-10-01T11:17:01.420945Z","shell.execute_reply.started":"2022-10-01T11:16:54.005947Z","shell.execute_reply":"2022-10-01T11:17:01.419193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import catalyst\ncatalyst.utils.plot_metrics(\n    logdir=logdir, \n    # specify which metrics we want to plot\n    metrics=[\"loss\", \"dice\", 'lr', '_base/lr']\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T11:17:01.423462Z","iopub.execute_input":"2022-10-01T11:17:01.424888Z","iopub.status.idle":"2022-10-01T11:17:02.976997Z","shell.execute_reply.started":"2022-10-01T11:17:01.424838Z","shell.execute_reply":"2022-10-01T11:17:02.975543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}