{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport json\nimport pytorch_lightning as pl\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn import model_selection\nimport torchvision.transforms as transforms\nimport torchvision.io \nfrom PIL import Image\nimport albumentations as alb","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:38.080944Z","iopub.execute_input":"2021-12-12T06:54:38.081659Z","iopub.status.idle":"2021-12-12T06:54:43.493281Z","shell.execute_reply.started":"2021-12-12T06:54:38.081568Z","shell.execute_reply":"2021-12-12T06:54:43.492451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:43.497364Z","iopub.execute_input":"2021-12-12T06:54:43.497590Z","iopub.status.idle":"2021-12-12T06:54:52.363882Z","shell.execute_reply.started":"2021-12-12T06:54:43.497552Z","shell.execute_reply":"2021-12-12T06:54:52.363040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformationType:\n    TORCHVISION = \"torchvision\"\n    ALB = \"albumentations\"\n\nclass Models:\n    RESNET34 = \"resnet34\"\n    RESNET50 = \"resnet50\"\n    RESNEXT50 = \"resnext50_32x4d\"    \n\nclass ImgStats:\n    IMAGENET_MEAN = [0.485, 0.456, 0.406]\n    IMAGENET_STD = [0.229, 0.224, 0.225]\n    CASSAVA_MEAN = [0.4303, 0.4967, 0.3135]\n    CASSAVA_STD = [0.2203, 0.2232, 0.2114]\n    \n# CONSTANTS\nclass Config:\n    NUM_CLASSES = 5\n    BATCH_SIZE = 48\n    NUM_FOLDS = 5\n    UNFREEZE_EPOCH_NO = 1\n    NUM_EPOCHS = 10\n    NUM_WORKERS = 2\n    INPUT_IMAGE_SIZE = (512,512)    \n    IMG_MEAN = ImgStats.IMAGENET_MEAN\n    IMG_STD = ImgStats.IMAGENET_STD\n    FAST_DEV_RUN = False\n    PRECISION = 16\n    IMG_ROOT_FOLDER = \"../input/cassava-leaf-disease-classification/\"\n    PATIENCE = 5    \n    SUBSET_ROWS_FRAC = 0.1\n    TRAIN_ON_SUBSET = True\n    RANDOM_SEED = 42\n    MODEL_TO_USE = Models.RESNEXT50\n    # model hyperparameters\n    MODEL_PARAMS = {    \n        \"drop_out\": 0.25,\n        \"lr\": 0.00036\n    }","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.365421Z","iopub.execute_input":"2021-12-12T06:54:52.366305Z","iopub.status.idle":"2021-12-12T06:54:52.375983Z","shell.execute_reply.started":"2021-12-12T06:54:52.366262Z","shell.execute_reply":"2021-12-12T06:54:52.374473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"with open(Config.IMG_ROOT_FOLDER + \"label_num_to_disease_map.json\") as label_mapping_file:\n    label_map = json.load(label_mapping_file)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.378600Z","iopub.execute_input":"2021-12-12T06:54:52.380063Z","iopub.status.idle":"2021-12-12T06:54:52.397234Z","shell.execute_reply.started":"2021-12-12T06:54:52.380023Z","shell.execute_reply":"2021-12-12T06:54:52.396550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_img_names = pd.read_csv(Config.IMG_ROOT_FOLDER + \"train.csv\")\n# Add a column with disease name\ndf_train_img_names[\"disease\"] = df_train_img_names.apply(\n    lambda row: label_map[str(row[\"label\"])], axis=1\n    )\ndf_train_img_names.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.399552Z","iopub.execute_input":"2021-12-12T06:54:52.400276Z","iopub.status.idle":"2021-12-12T06:54:52.704375Z","shell.execute_reply.started":"2021-12-12T06:54:52.400239Z","shell.execute_reply":"2021-12-12T06:54:52.703706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_img_names.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.705427Z","iopub.execute_input":"2021-12-12T06:54:52.705788Z","iopub.status.idle":"2021-12-12T06:54:52.714208Z","shell.execute_reply.started":"2021-12-12T06:54:52.705754Z","shell.execute_reply":"2021-12-12T06:54:52.713458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the disease distribution within the training samples. Distribution doesn't seem balanced with CMD outnumbering other disease types by a wide margin","metadata":{}},{"cell_type":"code","source":"# Disease distribution\ndisease_counts = df_train_img_names.disease.value_counts()\nprint(disease_counts)\nplt.bar(disease_counts.index, disease_counts.values)\nplt.xticks(rotation=90)\nplt.ylabel(\"Count\")\nplt.title(\"Disease count\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.715696Z","iopub.execute_input":"2021-12-12T06:54:52.715980Z","iopub.status.idle":"2021-12-12T06:54:52.935332Z","shell.execute_reply.started":"2021-12-12T06:54:52.715945Z","shell.execute_reply":"2021-12-12T06:54:52.934664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Config.TRAIN_ON_SUBSET:\n    print(f\"Selecting {Config.SUBSET_ROWS_FRAC * 100}% training data\")\n    df_train_img_names = df_train_img_names.sample(\n        frac=Config.SUBSET_ROWS_FRAC, \n        random_state=Config.RANDOM_SEED\n        )","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.936552Z","iopub.execute_input":"2021-12-12T06:54:52.937183Z","iopub.status.idle":"2021-12-12T06:54:52.949630Z","shell.execute_reply.started":"2021-12-12T06:54:52.937146Z","shell.execute_reply":"2021-12-12T06:54:52.948999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# split the training dataframe into kfolds for cross validation. We do this before any processing is done\n# on the data. We use stratified kfold if the target distribution is unbalanced\ndef strat_kfold_dataframe(df, target_col_name, num_folds=5):\n    # we create a new column called kfold and fill it with -1\n    df[\"kfold\"] = -1\n    # randomize of shuffle the rows of dataframe before splitting is done\n    df = df.sample(frac=1, random_state=42).reset_index(drop=True)\n    # get the target data\n    y = df[target_col_name].values\n    skf = model_selection.StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=Config.RANDOM_SEED)\n    for fold, (train_index, val_index) in enumerate(skf.split(X=df, y=y)):\n        df.loc[val_index, \"kfold\"] = fold    \n    return df     \n\ndf_train_img_names = strat_kfold_dataframe(df_train_img_names, target_col_name=\"label\")\ndf_train_img_names.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.950718Z","iopub.execute_input":"2021-12-12T06:54:52.951049Z","iopub.status.idle":"2021-12-12T06:54:52.975277Z","shell.execute_reply.started":"2021-12-12T06:54:52.951013Z","shell.execute_reply":"2021-12-12T06:54:52.974601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\n\n# https://kozodoi.me/python/deep%20learning/pytorch/tutorial/2021/03/08/image-mean-std.html\n# https://www.thoughtco.com/sum-of-squares-formula-shortcut-3126266\ndef get_imgs_mean_stddev(dl_img, axis=None):    \n    \"\"\"Get the mean and standard deviation for images in a dataset / mini-batch.\n    img batch is of shape BS * C * H * W \n    where BS = batch_size or no of training samples \n    C = 3 ( RGB channels ), H = height of image matrix, W = width of image matrix\n    Args:\n        dl_imgs ([DataLoader]): image data loader\n        axis ([tuple of ints], optional): Axis along which mean and std dev is to be calculated.\n        Defaults to None.\n    Returns:\n        [tuple]: tuple of tensors with mean and std.dev. of the imgs\n    \"\"\"\n    torch.manual_seed(Config.RANDOM_SEED)\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')    \n    # sum of pixel values along RGB channels\n    psum = torch.Tensor([0.0, 0.0, 0.0])\n    # sum of squares of pixel values along RGB channels\n    psum_sq = torch.Tensor([0.0, 0.0, 0.0])        \n    num_img = 0    \n    img_h, img_w = 0, 0    \n    count = 0\n    for img, label in tqdm.tqdm(dl_img): \n        if count == 0:            \n            img_h = img.shape[2]       \n            img_w = img.shape[3]\n        num_img += img.shape[0]            \n        psum += img.sum(axis=[0, 2, 3])        \n        img_sq = img.square()\n        psum_sq += img_sq.sum(axis=[0, 2, 3])\n        count += 1\n    # pixel count of single img (index 1 is the height and index 2 is width of img)\n    img_pixel_count = img_h * img_w      \n    total_pixel_count = num_img * img_pixel_count   \n    # mean of pixel values across the dataset        \n    total_mean = psum / total_pixel_count    \n    # variance of pixel values across the dataset\n    total_var = (psum_sq / total_pixel_count) - (total_mean.square())    \n    total_std = torch.sqrt(total_var)\n    return total_mean, total_std","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.976797Z","iopub.execute_input":"2021-12-12T06:54:52.977274Z","iopub.status.idle":"2021-12-12T06:54:52.986866Z","shell.execute_reply.started":"2021-12-12T06:54:52.977237Z","shell.execute_reply":"2021-12-12T06:54:52.986175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define a custom pytorch Dataset for training","metadata":{}},{"cell_type":"code","source":"# A dataset contains the logic to fetch, load and if required transform data to bring it to a format\n# that can be used by dataloaders for training\nclass CassavaImageDataset(Dataset):\n    def __init__(self, df, img_name_col, target_col, img_root_folder, transform=None, target_transform=None):\n        self.df = df\n        self.img_name_col = img_name_col\n        self.target_col = target_col\n        self.img_root_folder = img_root_folder\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __getitem__(self, index):\n        img_path = self.img_root_folder + \"/\" + self.df.loc[index, self.img_name_col]\n        img = np.array(Image.open(img_path))\n        img_label = self.df.loc[index, self.target_col]\n        if self.transform is not None:\n            augmented = self.transform(image=img)\n            img = augmented[\"image\"]\n        if self.target_transform is not None:\n            img_label = self.target_transform(img_label)\n        return img, img_label\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:52.988330Z","iopub.execute_input":"2021-12-12T06:54:52.988508Z","iopub.status.idle":"2021-12-12T06:54:52.999699Z","shell.execute_reply.started":"2021-12-12T06:54:52.988486Z","shell.execute_reply":"2021-12-12T06:54:52.998956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from albumentations.pytorch import ToTensorV2\n\ntrain_transform = alb.Compose([\n        alb.RandomResizedCrop(Config.INPUT_IMAGE_SIZE[0], Config.INPUT_IMAGE_SIZE[1]),        \n        alb.Transpose(p=0.5),                \n        alb.HorizontalFlip(p=0.5),\n        alb.VerticalFlip(p=0.5),\n        alb.ShiftScaleRotate(p=0.5),\n        alb.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5),        \n        alb.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5),\n        alb.Normalize(mean=Config.IMG_MEAN, std=Config.IMG_STD),\n        ToTensorV2()\n])\n\nval_transform = alb.Compose([\n        alb.CenterCrop(Config.INPUT_IMAGE_SIZE[0], Config.INPUT_IMAGE_SIZE[1]),\n        alb.Normalize(mean=Config.IMG_MEAN, std=Config.IMG_STD),\n        ToTensorV2()        \n])","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.001015Z","iopub.execute_input":"2021-12-12T06:54:53.001516Z","iopub.status.idle":"2021-12-12T06:54:53.015651Z","shell.execute_reply.started":"2021-12-12T06:54:53.001480Z","shell.execute_reply":"2021-12-12T06:54:53.014937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_fold_dls(fold, df_imgs):\n    df_train = df_imgs[df_imgs[\"kfold\"] != fold].reset_index(drop=True)\n    df_val = df_imgs[df_imgs[\"kfold\"] == fold].reset_index(drop=True)    \n    ds_train = CassavaImageDataset(\n        df_train, \n        img_name_col=\"image_id\",\n        target_col=\"label\",\n        img_root_folder=Config.IMG_ROOT_FOLDER + \"train_images\", \n        transform=train_transform,\n        target_transform=torch.as_tensor\n        )\n    ds_val = CassavaImageDataset(\n        df_val, \n        img_name_col=\"image_id\",\n        target_col=\"label\",\n        img_root_folder=Config.IMG_ROOT_FOLDER + \"train_images\", \n        transform=val_transform,\n        target_transform=torch.as_tensor\n        )        \n    dl_train = DataLoader(ds_train, batch_size=Config.BATCH_SIZE, shuffle=True, num_workers=Config.NUM_WORKERS)    \n    dl_val = DataLoader(ds_val, batch_size=Config.BATCH_SIZE, num_workers=Config.NUM_WORKERS)\n    return dl_train, dl_val, ds_train, ds_val","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.018586Z","iopub.execute_input":"2021-12-12T06:54:53.018763Z","iopub.status.idle":"2021-12-12T06:54:53.027531Z","shell.execute_reply.started":"2021-12-12T06:54:53.018741Z","shell.execute_reply":"2021-12-12T06:54:53.026810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl_train, dl_val, ds_train, ds_val = get_fold_dls(0, df_train_img_names)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.031376Z","iopub.execute_input":"2021-12-12T06:54:53.031625Z","iopub.status.idle":"2021-12-12T06:54:53.039523Z","shell.execute_reply.started":"2021-12-12T06:54:53.031601Z","shell.execute_reply":"2021-12-12T06:54:53.038841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"img mean calculated on entire training dataset = tensor([0.4303, 0.4967, 0.3135])<br>\nimg std calculated on entire training dataset = tensor([0.2203, 0.2232, 0.2114])","metadata":{}},{"cell_type":"code","source":"# display images along with their labels from a batch where images are in form of numpy arrays \n# if predictions are provided along with labels, these are displayed too\ndef show_batch(img_ds, num_items, num_rows, num_cols, predict_arr=None):\n    fig = plt.figure(figsize=(9, 6))\n    img_index = np.random.randint(0, len(img_ds)-1, num_items)\n    for index, img_index in enumerate(img_index):  # list first 9 images\n        img, lb = img_ds[img_index]        \n        ax = fig.add_subplot(num_rows, num_cols, index + 1, xticks=[], yticks=[])\n        if isinstance(img, torch.Tensor):\n            img = img.detach().numpy()\n        if isinstance(img, np.ndarray):\n            # the image data has RGB channels at dim 0, the shape of 3, 64, 64 needs to be 64, 64, 3 for display            \n            img = img.transpose(1, 2, 0)\n            ax.imshow(Image.fromarray(np.uint8(img)).convert('RGB'))        \n        if isinstance(lb, torch.Tensor):\n            # extract the label from label tensor\n            lb = lb.item()            \n        title = f\"Actual: {lb}\"\n        if predict_arr: \n            title += f\", Pred: {predict_arr[img_index]}\"        \n        ax.set_title(title)  ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.040769Z","iopub.execute_input":"2021-12-12T06:54:53.041144Z","iopub.status.idle":"2021-12-12T06:54:53.049717Z","shell.execute_reply.started":"2021-12-12T06:54:53.041108Z","shell.execute_reply":"2021-12-12T06:54:53.048955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_batch(ds_val, 3, 1, 3)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.051206Z","iopub.execute_input":"2021-12-12T06:54:53.051472Z","iopub.status.idle":"2021-12-12T06:54:53.595388Z","shell.execute_reply.started":"2021-12-12T06:54:53.051438Z","shell.execute_reply":"2021-12-12T06:54:53.594728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.models as models\nimport torch.nn as nn\nfrom torch.nn.functional import cross_entropy\nfrom torchmetrics.functional import accuracy\nimport timm\n\nclass ImageClassificationLitModel(pl.LightningModule):\n    def __init__(self, num_classes, hparams, model_to_use):\n        super().__init__()\n        print(f\"Using {model_to_use}\")\n        self.save_hyperparameters()\n        self.lr = hparams[\"lr\"]\n        self.num_classes = num_classes        \n        self.backbone, self.classifier = self.get_backbone_classifier(model_to_use, hparams[\"drop_out\"], num_classes) \n\n    @staticmethod\n    def get_backbone_classifier(model_to_use, drop_out, num_classes):\n        pt_model = timm.create_model(model_to_use, pretrained=True)\n        backbone = None\n        classifier = None\n        if model_to_use in [Models.RESNET34, Models.RESNET50, Models.RESNEXT50]:            \n            backbone = nn.Sequential(*list(pt_model.children())[:-1])\n            in_features = pt_model.fc.in_features\n            classifier = nn.Sequential(\n                nn.Linear(in_features, 512),\n                nn.ReLU(),\n                nn.Dropout(p=drop_out),\n                nn.Linear(512, 256),\n                nn.ReLU(),\n                nn.Dropout(p=drop_out),\n                nn.Linear(256, num_classes)\n            )                \n        return backbone, classifier\n\n    def forward(self, x):\n        features = self.backbone(x)\n        features = torch.flatten(features, 1)                \n        x = self.classifier(features)\n        return x\n\n    def configure_optimizers(self):\n        model_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, self.parameters()), lr=self.lr)\n        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model_optimizer, \"min\")        \n        return {\n            \"optimizer\": model_optimizer, \n            \"lr_scheduler\": {\n                \"scheduler\": lr_scheduler,\n                \"monitor\": \"val_loss\",\n                \"frequency\": 1\n            }\n        }\n\n    def training_step(self, batch, batch_idx):\n        X, y = batch\n        y_pred = self(X)\n        loss = cross_entropy(y_pred, y)\n        acc = accuracy(y_pred, y)\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        self.log(\"train_acc\", acc, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        return loss        \n\n    def validation_step(self, batch, batch_idx):\n        X, y = batch\n        y_pred = self(X)\n        val_loss = cross_entropy(y_pred, y)\n        val_acc = accuracy(y_pred, y)\n        self.log(\"val_loss\", val_loss, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        self.log(\"val_acc\", val_acc, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        return {\"loss\": val_loss, \"val_acc\": val_acc}","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:53.596412Z","iopub.execute_input":"2021-12-12T06:54:53.596645Z","iopub.status.idle":"2021-12-12T06:54:57.468001Z","shell.execute_reply.started":"2021-12-12T06:54:53.596615Z","shell.execute_reply":"2021-12-12T06:54:57.467270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.callbacks import ModelCheckpoint, BackboneFinetuning, EarlyStopping\n\n# For results reproducibility \n# sets seeds for numpy, torch, python.random and PYTHONHASHSEED.\npl.seed_everything(Config.RANDOM_SEED, workers=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.469403Z","iopub.execute_input":"2021-12-12T06:54:57.469640Z","iopub.status.idle":"2021-12-12T06:54:57.479173Z","shell.execute_reply.started":"2021-12-12T06:54:57.469607Z","shell.execute_reply":"2021-12-12T06:54:57.478353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning import LightningModule, Trainer\nfrom pytorch_lightning.callbacks import Callback\n\nclass MetricsAggCallback(Callback):\n    def __init__(self, metric_to_monitor, mode):\n        self.metric_to_monitor = metric_to_monitor\n        self.metrics = []\n        self.best_metric = None\n        self.mode = mode\n        self.best_metric_epoch = None\n\n    def on_epoch_end(self, trainer: Trainer, pl_module: LightningModule):\n        metric_value = trainer.callback_metrics[self.metric_to_monitor].cpu().detach().item()\n        val_loss = trainer.callback_metrics[\"val_loss\"].cpu().detach().item()\n        print(f\"metric {self.metric_to_monitor} = {metric_value}, val_loss={val_loss}\")\n        self.metrics.append(metric_value)\n        if self.mode == \"max\":\n            self.best_metric = max(self.metrics)\n            self.best_metric_epoch = self.metrics.index(self.best_metric)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.480693Z","iopub.execute_input":"2021-12-12T06:54:57.481223Z","iopub.status.idle":"2021-12-12T06:54:57.489432Z","shell.execute_reply.started":"2021-12-12T06:54:57.481186Z","shell.execute_reply":"2021-12-12T06:54:57.488757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(fold, dl_train, dl_val, fold_loss, fold_acc, find_lr=True):\n        fold_str = f\"fold{fold}\"\n        print(f\"Running training for {fold_str}\")\n        tb_logger = None\n        chkpt_file_name = \"best_model_{epoch}_{val_loss:.4f}\"        \n        multiplicative = lambda epoch: 1.5\n        backbone_finetuning = BackboneFinetuning(Config.UNFREEZE_EPOCH_NO, multiplicative, verbose=True)\n        early_stopping_callback = EarlyStopping(monitor=\"val_loss\", patience=Config.PATIENCE, mode=\"min\", verbose=True)\n    \n        if fold is not None:       \n            chkpt_file_name = fold_str + \"_\" + chkpt_file_name\n            tb_logger = pl.loggers.TensorBoardLogger(save_dir=\"logs\", version=fold_str)\n        else:\n            tb_logger = pl.loggers.TensorBoardLogger(save_dir=\"logs\")        \n        cassava_model = ImageClassificationLitModel(\n            num_classes=Config.NUM_CLASSES, \n            hparams=Config.MODEL_PARAMS,        \n            model_to_use=Config.MODEL_TO_USE\n            )      \n        loss_chkpt_callback = ModelCheckpoint(dirpath=\"./model\", verbose=True, monitor=\"val_loss\", mode=\"min\", filename=chkpt_file_name)\n        acc_chkpt_callback = MetricsAggCallback(metric_to_monitor=\"val_acc\", mode=\"max\")\n        trainer = pl.Trainer(\n            gpus=1,\n            # For results reproducibility \n            deterministic=True,\n            auto_select_gpus=True,\n            progress_bar_refresh_rate=20,\n            max_epochs=Config.NUM_EPOCHS,\n            logger=tb_logger,\n            auto_lr_find=True,    \n            precision=Config.PRECISION,    \n            weights_summary=None, \n            fast_dev_run=Config.FAST_DEV_RUN,                   \n            callbacks=[loss_chkpt_callback, acc_chkpt_callback, backbone_finetuning, early_stopping_callback]\n        )\n        if find_lr:\n            trainer.tune(model=cassava_model, train_dataloaders=dl_train)\n            print(cassava_model.lr)\n        trainer.fit(cassava_model, train_dataloaders=dl_train, val_dataloaders=dl_val)                \n        if not Config.FAST_DEV_RUN:\n            fold_loss.append(loss_chkpt_callback.best_model_score.cpu().detach().item())\n            fold_acc.append(acc_chkpt_callback.best_metric)\n            print(f\"Loss for {fold_str} = {fold_loss[fold]}, accuracy = {fold_acc[fold]}\")\n        del trainer, cassava_model, backbone_finetuning, early_stopping_callback, acc_chkpt_callback, loss_chkpt_callback ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.492261Z","iopub.execute_input":"2021-12-12T06:54:57.492439Z","iopub.status.idle":"2021-12-12T06:54:57.505848Z","shell.execute_reply.started":"2021-12-12T06:54:57.492417Z","shell.execute_reply":"2021-12-12T06:54:57.505165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statistics\n\ndef print_exp_statistics(fold_loss, fold_acc):\n    print(\"Loss across folds\")\n    print(fold_loss)\n    print(\"Accuracy across folds\")\n    print(fold_acc)\n    #mean_loss = statistics.mean(fold_loss)\n    #mean_acc = statistics.mean(fold_acc)\n    #std_loss = statistics.stdev(fold_loss)\n    #std_acc = statistics.stdev(fold_acc)\n    #print(f\"mean loss across folds = {mean_loss}, loss stdev across fold = {std_loss}\")\n    #print(f\"mean accuracy across folds = {mean_acc}, accuracy stdev across fold = {std_acc}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.507356Z","iopub.execute_input":"2021-12-12T06:54:57.507848Z","iopub.status.idle":"2021-12-12T06:54:57.519548Z","shell.execute_reply.started":"2021-12-12T06:54:57.507810Z","shell.execute_reply":"2021-12-12T06:54:57.518863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_experiment():\n    find_lr = True\n    fold_loss = []\n    fold_acc = []\n    for fold in range(Config.NUM_FOLDS):\n        dl_train, dl_val, ds_train, ds_val = get_fold_dls(fold, df_train_img_names)\n        run_training(fold, dl_train, dl_val, fold_loss, fold_acc, find_lr)\n        break  \n    print_exp_statistics(fold_loss, fold_acc)       ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.522379Z","iopub.execute_input":"2021-12-12T06:54:57.522881Z","iopub.status.idle":"2021-12-12T06:54:57.529935Z","shell.execute_reply.started":"2021-12-12T06:54:57.522856Z","shell.execute_reply":"2021-12-12T06:54:57.529223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Config for experiment 1\n\nConfig.MODEL_TO_USE = Models.RESNET50\nConfig.NUM_EPOCHS = 10\nConfig.IMG_MEAN = ImgStats.IMAGENET_MEAN\nConfig.IMG_STD = ImgStats.IMAGENET_STD\nConfig.INPUT_IMAGE_SIZE = (512,512)\nConfig.BATCH_SIZE = 64\n\nrun_experiment()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T06:54:57.530755Z","iopub.execute_input":"2021-12-12T06:54:57.532983Z","iopub.status.idle":"2021-12-12T07:10:08.292786Z","shell.execute_reply.started":"2021-12-12T06:54:57.532942Z","shell.execute_reply":"2021-12-12T07:10:08.291511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Config for experiment 2\n\nConfig.MODEL_TO_USE = Models.RESNEXT50\nConfig.NUM_EPOCHS = 10\nConfig.IMG_MEAN = ImgStats.IMAGENET_MEAN\nConfig.IMG_STD = ImgStats.IMAGENET_STD\nConfig.INPUT_IMAGE_SIZE = (512,512)\nConfig.BATCH_SIZE = 48\n\nrun_experiment()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T07:10:08.294658Z","iopub.execute_input":"2021-12-12T07:10:08.294976Z","iopub.status.idle":"2021-12-12T07:28:17.259675Z","shell.execute_reply.started":"2021-12-12T07:10:08.294932Z","shell.execute_reply":"2021-12-12T07:28:17.258546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = ResnetTL_LitModel.load_from_checkpoint(\"./model/cassava_best_model.ckpt\",     \n#                                                 num_classes=5)\n# model.to(\"cuda\")\n# model.eval()\n\n# incorrect = 0\n# total = 0\n# predicted_labels_incorrect = []\n# labels_incorrect = []\n# with torch.no_grad():\n#     counter=0\n#     for imgs, labels in tqdm.tqdm(dm.val_dataloader()):                \n#         predicted_cuda_labels = torch.argmax(model(imgs.to(\"cuda\")), dim=1)\n#         predicted_labels = predicted_cuda_labels.cpu().detach()\n#         total += labels.shape[0]\n#         correct_pred = predicted_labels == labels\n#         incorrect_pred = ~correct_pred\n#         num_incorrect_pred = incorrect_pred.sum()\n#         incorrect += int(num_incorrect_pred)\n#         if num_incorrect_pred > 0:\n#             predicted_labels_incorrect.append(predicted_labels[incorrect_pred].numpy())\n#             labels_incorrect.append(labels[incorrect_pred].numpy())\n# print(f'Total no. of images in validation set: {total}')\n# print(f'Incorrectly classified images in validation set: {incorrect}')\n# accuracy = ((total-incorrect) / total) * 100        \n# print(f\"Accuracy: {accuracy}%\")","metadata":{"execution":{"iopub.status.busy":"2021-12-12T07:28:17.261331Z","iopub.execute_input":"2021-12-12T07:28:17.261858Z","iopub.status.idle":"2021-12-12T07:28:17.266978Z","shell.execute_reply.started":"2021-12-12T07:28:17.261813Z","shell.execute_reply":"2021-12-12T07:28:17.266025Z"},"trusted":true},"execution_count":null,"outputs":[]}]}