{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Goal \n\nYou are provided with a large set of retina images taken using fundus photography under a variety of imaging conditions.\nFor further details, read instructions here: https://www.kaggle.com/c/aptos2019-blindness-detection \n\n\nClassify each image into severity category :\n\n```\n0 - No DR\n\n1 - Mild\n\n2 - Moderate\n\n3 - Severe\n\n4 - Proliferative DR\n```"},{"metadata":{},"cell_type":"markdown","source":"lets start with installing torch image models that will load any pretrained models we need in this notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n\n# general imports\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt \n%matplotlib inline\nimport seaborn as sns\nsns.set()\n\nimport cv2\nimport os  \nfrom PIL import Image \nfrom pprint import pprint\n\n# torch and torchvision\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torchvision.transforms as transforms\n\n# load pretrained models \nimport timm\n\n# catalyst for training and metrics\nfrom catalyst.utils import split_dataframe_train_test\nfrom catalyst.dl.callbacks import AccuracyCallback\nfrom catalyst.dl import SupervisedRunner","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameters\n\nThe following function contains dict that has all the paramters used to train in this notebook. Change these values here and re-run to see updated training"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def config():\n    cfg = {\n        # raw csv data\n        'train_csv_path': '/kaggle/input/aptos2019-blindness-detection/train.csv',\n        'test_csv_path': '/kaggle/input/aptos2019-blindness-detection/test.csv',\n        # images path\n        'img_root': '/kaggle/input/aptos2019-blindness-detection/train_images/',\n        'test_img_root': '/kaggle/input/aptos2019-blindness-detection/test_images/',\n        # backend architecture, features are extracted from this\n        'arch': 'resnext50_32x4d',\n        # training parameters \n        'random_state': 1,\n        'num_classes': 5,\n        'test_size': 0.2,\n        'input_size': 512,\n        'freeze': True,\n        'lr': 3e-4,\n        'logdir': '/kaggle/working/logs/',\n        'device': None,\n        'batch_size': 8,\n        'test_batch_size': 2,\n        'num_epochs': 7,\n        # logging \n        'verbose': True,\n        'check': False,  # set this true to run for 3 epochs only\n        # data labels\n        'class_names': ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative']\n\n    }\n    return cfg\n\ncfg = config()\ncfg['device'] = torch.device(\n        \"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Parameters for Training:\")\npprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Visualization\nLets see data and how the labels are distributed across classes. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(cfg['train_csv_path'])\ntrain_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above table, `id_code` is the image filename and `diagnosis` is category into which we have to predict. "},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(12,10))\nsns.distplot(train_df['diagnosis'], bins=5, kde=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above plot, category index 0 and 2 has relatively large number of samples. This can induce bias in out training model. In order to mitigate this, lets try the strategy of selectively sampling. For categories 0 & 2, they will be undersamples while keeping the sampling rate same for all the other categories. "},{"metadata":{},"cell_type":"markdown","source":"## Data Balancing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def balance_data(csv_path: str, test_size: float = 0.2, random_state: int = 123):\n    df = pd.read_csv(csv_path)\n    # first class has large number of samples as compares to others\n    # one way to balance is by sampling smaller amount of data\n    class_0 = df[df['diagnosis'] == 0]\n    class_0 = class_0.sample(400)\n    class_0_train, class_0_test = split_dataframe_train_test(\n        class_0, test_size=test_size, random_state=random_state)\n    df_train = class_0_train\n    df_test = class_0_test\n\n    class_1 = df[df['diagnosis'] == 1]\n    class_1_train, class_1_test = split_dataframe_train_test(\n        class_1, test_size=test_size, random_state=random_state)\n    df_train = df_train.append(class_1_train)\n    df_test = df_test.append(class_1_test)\n\n    # sub sampling data for Moderate category\n    class_2 = df[df['diagnosis'] == 2]\n    class_2 = class_2.sample(400)\n    class_2_train, class_2_test = split_dataframe_train_test(\n        class_2, test_size=test_size, random_state=random_state)\n    df_train = df_train.append(class_2_train)\n    df_test = df_test.append(class_2_test)\n\n    class_3 = df[df['diagnosis'] == 3]\n    class_3_train, class_3_test = split_dataframe_train_test(\n        class_3, test_size=test_size, random_state=random_state)\n    df_train = df_train.append(class_3_train)\n    df_test = df_test.append(class_3_test)\n\n    class_4 = df[df['diagnosis'] == 4]\n    class_4_train, class_4_test = split_dataframe_train_test(\n        class_4, test_size=test_size, random_state=random_state)\n    df_train = df_train.append(class_4_train)\n    df_test = df_test.append(class_4_test)\n\n    return df_train, df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df = balance_data(cfg['train_csv_path'])\nprint(\"Training Samples:\")\nprint(\"No DR:\", len(train_df[train_df['diagnosis']==0]))\nprint(\"Mild:\", len(train_df[train_df['diagnosis']==1]))\nprint(\"Moderate:\", len(train_df[train_df['diagnosis']==2]))\nprint(\"Severe:\", len(train_df[train_df['diagnosis']==3]))\nprint(\"Proliferative DR:\", len(train_df[train_df['diagnosis']==4]))\nprint(\"\\nTest Samples:\")\nprint(\"No DR:\", len(test_df[test_df['diagnosis']==0]))\nprint(\"Mild:\", len(test_df[test_df['diagnosis']==1]))\nprint(\"Moderate:\", len(test_df[test_df['diagnosis']==2]))\nprint(\"Severe:\", len(test_df[test_df['diagnosis']==3]))\nprint(\"Proliferative DR:\", len(test_df[test_df['diagnosis']==4]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(12,10))\nsns.distplot(train_df['diagnosis'], bins=5, kde=False);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(12,10))\nsns.distplot(test_df['diagnosis'], bins=5, kde=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sample Plots"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_sample(root:str,filename:str):\n    img = cv2.imread(os.path.join(root, filename+'.png'))\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_samples(df:pd.DataFrame, idx:int=0):\n    filename = df.iloc[idx]['id_code']\n    label = df.iloc[idx]['diagnosis']\n    img = read_sample(cfg['img_root'],filename)\n    print(f\"Image:{img.shape}\")\n    fig = plt.subplots(nrows=1, ncols=1, figsize=(10,10))\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\n    plt.title(f\"Diagnosis:{label}\")\n    plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_samples(train_df, 789); plot_samples(train_df, 432)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above figure, a sample data image is shown as well as corresponding labels. To view more samples, change the index value in function `plot_samples` "},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AptosDataset(Dataset):\n    \"\"\"Retrieves each data item for use with dataloaders\"\"\"\n    def __init__(self,\n                 img_root: str,\n                 df: pd.DataFrame,\n                 img_transforms: transforms = None,\n                 is_train: bool = True\n                 ):\n\n        self.df = df\n        self.img_root = img_root\n        self.img_transforms = img_transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n        row = self.df.iloc[idx]\n        filename = row['id_code']\n        target = int(row['diagnosis'])\n        img = Image.open(os.path.join(\n            self.img_root, filename+'.png')).convert('RGB')\n        img = np.asarray(img)\n        if self.img_transforms is not None:\n            augmented = self.img_transforms(image=img)\n            img = augmented['image']\n        return img, np.asarray(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as albu\nfrom albumentations.pytorch import ToTensor\n\n\n\ndef pre_transforms(image_size=512):\n    # Convert the image to a square of size image_size x image_size\n    # (keeping aspect ratio)\n    result = [\n        albu.LongestMaxSize(max_size=image_size),\n        albu.PadIfNeeded(image_size, image_size, border_mode=2)\n    ]\n    \n    return result\n\ndef hard_transforms():\n    result = [\n        # Random shifts, stretches and turns with a 50% probability\n        albu.ShiftScaleRotate( \n            shift_limit=0.1,\n            scale_limit=0.1,\n            rotate_limit=15,\n            border_mode=0,\n            p=0.5\n        ),\n        # add random brightness and contrast, 30% prob\n        albu.RandomBrightnessContrast(\n            brightness_limit=0.2, contrast_limit=0.2, p=0.3\n        ),\n        # Random gamma changes with a 30% probability\n        albu.RandomGamma(gamma_limit=(85, 115), p=0.3),\n        # Randomly changes the hue, saturation, and color value of the input image \n        albu.HueSaturationValue(p=0.3),\n        albu.JpegCompression(quality_lower=80),\n    ]\n    \n    return result\n\ndef post_transforms():\n    # we use ImageNet image normalization\n    # and convert it to torch.Tensor\n    return [albu.Normalize(), ToTensor()]\n\ndef compose(transforms_to_compose):\n    # combine all augmentations into one single pipeline\n    result = albu.Compose([\n      item for sublist in transforms_to_compose for item in sublist\n    ])\n    return result\n\ndef get_transforms():\n    \n    train_transforms = compose([\n                        pre_transforms(), \n                        hard_transforms(), \n                        post_transforms()\n    ])\n    \n    val_transforms = compose([pre_transforms(), post_transforms()])\n    \n    return train_transforms, val_transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transforms, test_transforms = get_transforms()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = AptosDataset(\n        img_root=cfg['img_root'],\n        df=train_df,\n        img_transforms=train_transforms,\n        is_train=True,\n    )\n\ntest_dataset = AptosDataset(\n        img_root=cfg['img_root'],\n        df=test_df,\n        img_transforms=test_transforms,\n        is_train=False,\n    )\nprint(f\"Training set size:{len(train_dataset)}, Test set size:{len(test_dataset)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_dataset, cfg['batch_size'], shuffle=True, num_workers=1)\ntest_loader = DataLoader(test_dataset, cfg['test_batch_size'], shuffle=False, num_workers=1)\n\nloaders = {\n        'train': train_loader,\n        'valid': test_loader\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model\nLets setup model that will extract features from a pre-trained model on Imagenet. Here, the default choice is resnet-101 for feature extractor and then there are additional Linear layers with dropout in between to learn this data specific features for classification. "},{"metadata":{"trusted":true},"cell_type":"code","source":"class AptosModelV2(nn.Module):\n    def __init__(self, \n                 arch:str='resnet101', \n                 z_dims:int=2048, \n                 nb_classes:int=5,\n                 drop:float=0.5):\n        super(AptosModelV2, self).__init__()\n        self.model = timm.create_model(arch, pretrained=True,drop_rate=drop)\n        self.model.reset_classifier(num_classes = nb_classes)\n        \n    def forward(self, x):\n        return self.model(x)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = AptosModelV2(arch=cfg['arch'])\nmodel.train();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=3e-4)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tried launching tensorboard but doesn't work in browser\n# you can launch on local machine or colab\n%load_ext tensorboard\n%tensorboard --logdir cfg['logdir']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"runner = SupervisedRunner(device=cfg['device'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following code launches training and will display metrics log. In order to test this, we can run for few epochs instead of 50 epochs. You can set the flag `check` to true to run for 3 epochs only. "},{"metadata":{"trusted":true},"cell_type":"code","source":"runner.train(\n    model=model,\n    criterion=criterion,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    loaders=loaders,\n    \n    callbacks=[\n        AccuracyCallback(\n            num_classes=cfg['num_classes'],\n            threshold=0.5,\n            activation=\"Sigmoid\"\n        ),\n    ],\n    logdir=cfg['logdir'],\n    num_epochs=cfg['num_epochs'],\n    verbose=cfg['verbose'],\n    # set this true to run for 3 epochs only\n    check=cfg['check'],\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.dl import utils\n\nutils.plot_metrics(\n    logdir=cfg['logdir'], \n    metrics=[\"loss\", \"accuracy01\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation\n\nNow, lets create classification report based on our test dataset. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_evaluation():\n    # given model and valid dataset \n    # iterate over dataset and compute prediction\n    y_true = []\n    y_pred = []\n    test_size = len(test_dataset)\n    model.eval()\n    for i in range(test_size):\n        img_tensor = test_dataset[i][0].unsqueeze(0,)\n        with torch.no_grad():\n            pred = torch.sigmoid(model(img_tensor.to(cfg['device']))).squeeze().cpu()\n            _,output = torch.topk(pred,1)\n            output = output.numpy()[0]\n        label = test_dataset[i][1].item()\n        y_true.append(label)\n        y_pred.append(output)\n    \n    return y_true, y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_true, test_pred = run_evaluation()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(test_true, test_pred, target_names=cfg['class_names']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above table, the precision score for each category. In the bottom there is accuracy, average precision when computed `macro` and `weighted` average precision. "},{"metadata":{},"cell_type":"markdown","source":"## Submission\n\nLets submit our predicition for held out set (this does not have labels)."},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_on_held_out(csv_path, img_root, img_transforms):\n    # given model and valid dataset \n    # iterate over dataset and compute prediction\n    \n    df = pd.read_csv(csv_path)\n    test_size = len(df)\n    print(f\"Size: {test_size}\")\n    y_pred = {}\n    model.eval()\n    for idx,row in df.iterrows():\n        filename = row['id_code']\n        # load and transform input imate\n        img = Image.open(os.path.join(\n            img_root, filename+'.png')).convert('RGB')\n        img = np.asarray(img)\n        augmented = img_transforms(image=img)\n        img_tensor = augmented['image']\n        img_tensor = img_tensor.unsqueeze(0,)\n        \n        # run prediction\n        with torch.no_grad():\n            pred = torch.sigmoid(model(img_tensor.to(cfg['device']))).squeeze().cpu()\n            _,output = torch.topk(pred,1)\n            output = output.numpy()[0]\n        # store results\n        y_pred[filename] = output\n    \n    return y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_dict = run_on_held_out(cfg['test_csv_path'], cfg['test_img_root'], test_transforms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame.from_dict(submission_dict, orient='index', columns=['diagnosis'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.index.name = 'id_code'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}