{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EFFICIENT NET B4 MODEL","metadata":{}},{"cell_type":"markdown","source":"The EfficientNet B4 model from the tf_efficientnet_b4_ns architecture, is a variant of EfficientNet designed for high performance with non-separable (NS) convolution layers, providing balanced depth, width, and input resolution. The model employs a 5-fold cross-validation approach, with a training batch size of 16 and a validation batch size of 32, ensuring comprehensive evaluation and learning. Training is set for 5 epochs, balancing the time and performance. The learning rate is set to 1e-4, with a minimum of 1e-6, using a Cosine Annealing schedule with a 10-cycle length to manage learning progression. To prevent overfitting, the weight decay was set to 1e-6, and batch accumulation with an iteration value of 2 allows for effective larger-batch training. \nInput images are resized to 512x512 pixels, providing a balance between detail and computational efficiency. To balance the classes in the dataset, image augmentation was performed to create multiple variants of the existing images to eliminate bias towards a particular class. Images were modified by resizing, flips, rotations, shifts and colour alterations to ensure variability. These techniques ensure that the model performs well on unseen data and results in optimal performance.\n","metadata":{}},{"cell_type":"markdown","source":"## IMPORTS","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-27T21:06:46.303877Z","iopub.execute_input":"2024-04-27T21:06:46.304255Z","iopub.status.idle":"2024-04-27T21:06:46.645837Z","shell.execute_reply.started":"2024-04-27T21:06:46.304224Z","shell.execute_reply":"2024-04-27T21:06:46.645015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:46.647252Z","iopub.execute_input":"2024-04-27T21:06:46.647616Z","iopub.status.idle":"2024-04-27T21:06:53.209142Z","shell.execute_reply.started":"2024-04-27T21:06:46.647591Z","shell.execute_reply":"2024-04-27T21:06:53.208148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter Declaration","metadata":{}},{"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'epochs': 5,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 4,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0'\n}","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.210176Z","iopub.execute_input":"2024-04-27T21:06:53.210446Z","iopub.status.idle":"2024-04-27T21:06:53.215688Z","shell.execute_reply.started":"2024-04-27T21:06:53.210422Z","shell.execute_reply":"2024-04-27T21:06:53.214829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Inspection","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.218042Z","iopub.execute_input":"2024-04-27T21:06:53.218306Z","iopub.status.idle":"2024-04-27T21:06:53.267615Z","shell.execute_reply.started":"2024-04-27T21:06:53.218284Z","shell.execute_reply":"2024-04-27T21:06:53.266747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.268530Z","iopub.execute_input":"2024-04-27T21:06:53.268817Z","iopub.status.idle":"2024-04-27T21:06:53.280982Z","shell.execute_reply.started":"2024-04-27T21:06:53.268793Z","shell.execute_reply":"2024-04-27T21:06:53.280060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.282103Z","iopub.execute_input":"2024-04-27T21:06:53.282391Z","iopub.status.idle":"2024-04-27T21:06:53.296441Z","shell.execute_reply.started":"2024-04-27T21:06:53.282367Z","shell.execute_reply":"2024-04-27T21:06:53.295657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"#Setting a seed ensures reproducibility of experiments by making random number generation deterministic\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \n#Retrieve image from dataset\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.297405Z","iopub.execute_input":"2024-04-27T21:06:53.297660Z","iopub.status.idle":"2024-04-27T21:06:53.739436Z","shell.execute_reply.started":"2024-04-27T21:06:53.297638Z","shell.execute_reply":"2024-04-27T21:06:53.738549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Generates random bounding boxes within a given image size,\ndef rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n#Dataset is stored as an object with this class type to be processed\nclass CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True, \n                 one_hot_label=False,\n                 do_fmix=False, \n                 fmix_params={\n                     'alpha': 1., \n                     'decay_power': 3., \n                     'shape': (CFG['img_size'], CFG['img_size']),\n                     'max_soft': True, \n                     'reformulate': False\n                 },\n                 do_cutmix=False,\n                 cutmix_params={\n                     'alpha': 1,\n                 }\n                ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n        \n        self.output_label = output_label\n        self.one_hot_label = one_hot_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            \n            if one_hot_label is True:\n                self.labels = np.eye(self.df['label'].max()+1)[self.labels]\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image_id']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():       \n                lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']),0.6,0.7)\n                \n                # Make mask, get mean / std\n                mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])\n                mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])\n    \n                fmix_ix = np.random.choice(self.df.index, size=1)[0]\n                fmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))\n\n                if self.transforms:\n                    fmix_img = self.transforms(image=fmix_img)['image']\n\n                mask_torch = torch.from_numpy(mask)\n                \n                # mix image\n                img = mask_torch*img+(1.-mask_torch)*fmix_img\n\n\n                # mix target\n                rate = mask.sum()/CFG['img_size']/CFG['img_size']\n                target = rate*target + (1.-rate)*self.labels[fmix_ix]\n        \n        if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n            with torch.no_grad():\n                cmix_ix = np.random.choice(self.df.index, size=1)[0]\n                cmix_img  = get_img(\"{}/{}\".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))\n                if self.transforms:\n                    cmix_img = self.transforms(image=cmix_img)['image']\n                    \n                lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']),0.3,0.4)\n                bbx1, bby1, bbx2, bby2 = rand_bbox((CFG['img_size'], CFG['img_size']), lam)\n\n                img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n                rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (CFG['img_size'] * CFG['img_size']))\n                target = rate*target + (1.-rate)*self.labels[cmix_ix]\n                \n                            \n        # label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.740565Z","iopub.execute_input":"2024-04-27T21:06:53.740843Z","iopub.status.idle":"2024-04-27T21:06:53.764632Z","shell.execute_reply.started":"2024-04-27T21:06:53.740818Z","shell.execute_reply":"2024-04-27T21:06:53.763829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transformation and Image Augmentation Functions","metadata":{}},{"cell_type":"code","source":"from albumentations import *\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.765841Z","iopub.execute_input":"2024-04-27T21:06:53.766104Z","iopub.status.idle":"2024-04-27T21:06:53.925046Z","shell.execute_reply.started":"2024-04-27T21:06:53.766078Z","shell.execute_reply":"2024-04-27T21:06:53.924165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Class and Functions","metadata":{}},{"cell_type":"markdown","source":"Image Classifier class for the EfficientNet B4 Model","metadata":{}},{"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.928946Z","iopub.execute_input":"2024-04-27T21:06:53.929469Z","iopub.status.idle":"2024-04-27T21:06:53.935536Z","shell.execute_reply.started":"2024-04-27T21:06:53.929445Z","shell.execute_reply":"2024-04-27T21:06:53.934570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/'):\n    \n    from catalyst.data.sampler import BalanceClassSampler\n    \n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms(), output_label=True, one_hot_label=False, do_fmix=False, do_cutmix=False)\n    valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=False,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n        #sampler=BalanceClassSampler(labels=train_['label'].values, mode=\"downsampling\")\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    return train_loader, val_loader\n\nimport torch\nfrom torch.cuda.amp import autocast\nfrom tqdm import tqdm\n\ndef train_one_epoch(\n    epoch,\n    model,\n    loss_fn,\n    optimizer,\n    train_loader,\n    device,\n    scaler,\n    CFG,\n    scheduler=None,\n    schd_batch_update=False\n):\n    model.train()\n    running_loss = 0\n    correct_predictions = 0\n    total_samples = 0\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        with autocast():\n            image_preds = model(imgs)\n            loss = loss_fn(image_preds, image_labels)\n\n        scaler.scale(loss).backward()\n\n        running_loss += loss.item() * image_labels.shape[0]\n        total_samples += image_labels.shape[0]\n\n        correct_predictions += torch.sum(\n            torch.argmax(image_preds, dim=1) == image_labels\n        ).item()\n\n        if (step + 1) % CFG['accum_iter'] == 0 or (step + 1) == len(train_loader):\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n\n            if scheduler is not None and schd_batch_update:\n                scheduler.step()\n\n        if (step + 1) % CFG['verbose_step'] == 0 or (step + 1) == len(train_loader):\n            pbar.set_description(f'epoch {epoch} loss: {running_loss / total_samples:.4f}')\n\n    if scheduler is not None and not schd_batch_update:\n        scheduler.step()\n\n    avg_loss = running_loss / total_samples\n    accuracy = correct_predictions / total_samples\n    return avg_loss, accuracy\n\ndef valid_one_epoch(\n    epoch,\n    model,\n    loss_fn,\n    val_loader,\n    device,\n    CFG,\n    scheduler=None,\n    schd_loss_update=False\n):\n    model.eval()\n    loss_sum = 0\n    total_samples = 0\n    correct_predictions = 0\n    image_preds_all = []\n    image_targets_all = []\n\n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        with torch.no_grad():\n            image_preds = model(imgs)\n            loss = loss_fn(image_preds, image_labels)\n\n            image_preds_all.append(\n                torch.argmax(image_preds, dim=1).detach().cpu().numpy()\n            )\n            image_targets_all.append(image_labels.detach().cpu().numpy())\n        \n        loss_sum += loss.item() * image_labels.shape[0]\n        total_samples += image_labels.shape[0]\n        correct_predictions += torch.sum(\n            torch.argmax(image_preds, dim=1) == image_labels\n        ).item()\n\n        if (step + 1) % CFG['verbose_step'] == 0 or (step + 1) == len(val_loader):\n            pbar.set_description(f'epoch {epoch} loss: {loss_sum / total_samples:.4f}')\n\n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n\n    accuracy = (image_preds_all == image_targets_all).mean()\n    avg_loss = loss_sum / total_samples\n\n    print(f'Validation multi-class accuracy = {accuracy:.4f}')\n    \n    if scheduler is not None:\n        if schd_loss_update:\n            scheduler.step(avg_loss)\n        else:\n            scheduler.step()\n    \n    return avg_loss, accuracy\n","metadata":{"execution":{"iopub.status.busy":"2024-04-27T22:35:35.915904Z","iopub.execute_input":"2024-04-27T22:35:35.916344Z","iopub.status.idle":"2024-04-27T22:35:35.943181Z","shell.execute_reply.started":"2024-04-27T22:35:35.916308Z","shell.execute_reply":"2024-04-27T22:35:35.942211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Class to check loss obtained for each training iteration","metadata":{}},{"cell_type":"code","source":"# reference: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733\nclass MyCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean'):\n        super().__init__(weight=weight, reduction=reduction)\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2024-04-27T21:06:53.960678Z","iopub.execute_input":"2024-04-27T21:06:53.960967Z","iopub.status.idle":"2024-04-27T21:06:53.975087Z","shell.execute_reply.started":"2024-04-27T21:06:53.960944Z","shell.execute_reply":"2024-04-27T21:06:53.974240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"code","source":"from torch.cuda.amp import GradScaler\n# Initialize scaler for mixed-precision training\nscaler = GradScaler()\n\ntrain_losses = []\nval_losses = []\ntrain_accuracies = []\nval_accuracies = []\n\nif __name__ == '__main__':\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train, trn_idx, val_idx, data_root='../input/cassava-leaf-disease-classification/train_images/')\n\n        device = torch.device(CFG['device'])\n        \n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        \n        loss_tr = nn.CrossEntropyLoss().to(device) #MyCrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        \n        # Training loop with metric collection\n        for epoch in range(CFG['epochs']):\n            train_loss, train_acc = train_one_epoch(epoch,model,loss_tr,optimizer,train_loader,device,scaler=scaler,CFG=CFG,scheduler=scheduler,schd_batch_update=False\n            )\n            train_losses.append(train_loss)\n            train_accuracies.append(train_acc)\n\n            with torch.no_grad():\n                val_loss, val_acc = valid_one_epoch(\n                    epoch,\n                    model,\n                    loss_fn,\n                    val_loader,\n                    device,\n                    CFG=CFG,\n                    scheduler=None,\n                    schd_loss_update=False\n                )\n                val_losses.append(val_loss)\n                val_accuracies.append(val_acc)\n            torch.save(model.state_dict(), '{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n\n\n        del model, optimizer, train_loader, val_loader, scaler, scheduler #clean memory\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T22:38:57.899839Z","iopub.execute_input":"2024-04-27T22:38:57.900522Z","iopub.status.idle":"2024-04-28T03:48:16.405165Z","shell.execute_reply.started":"2024-04-27T22:38:57.900489Z","shell.execute_reply":"2024-04-28T03:48:16.404019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Results and Visualization","metadata":{}},{"cell_type":"code","source":"# Plotting the metrics\nimport matplotlib.pyplot as plt\n# Ensure the lengths match\nnum_epochs = len(train_losses)  # or min(len(train_losses), len(val_losses))\n\n# Ensure `epochs` array matches the length of metric arrays\nepochs = range(0, num_epochs)\n\n# Plot the corrected data\nplt.figure(figsize=(12, 6))\nplt.plot(epochs, train_losses[:num_epochs], 'b-', label='Training Loss')\nplt.plot(epochs, val_losses[:num_epochs], 'r-', label='Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Training and Validation Loss')\nplt.legend()\nplt.show()\n\nplt.figure(figsize=(12, 6))\nplt.plot(epochs, train_accuracies[:num_epochs], 'b-', label='Training Accuracy')\nplt.plot(epochs, val_accuracies[:num_epochs], 'r-', label='Validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.legend()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-28T03:50:03.744613Z","iopub.execute_input":"2024-04-28T03:50:03.745024Z","iopub.status.idle":"2024-04-28T03:50:04.357365Z","shell.execute_reply.started":"2024-04-28T03:50:03.744985Z","shell.execute_reply":"2024-04-28T03:50:04.356140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Model from Saved State","metadata":{}},{"cell_type":"markdown","source":"This step is important as we need to ensure the model can be loaded so we can load it into the inference notebook","metadata":{}},{"cell_type":"code","source":"state_dict = torch.load('/kaggle/working/tf_efficientnet_b4_ns_fold_4_4', map_location=torch.device('cpu'))","metadata":{"execution":{"iopub.status.busy":"2024-04-28T03:51:30.398471Z","iopub.execute_input":"2024-04-28T03:51:30.399097Z","iopub.status.idle":"2024-04-28T03:51:30.477727Z","shell.execute_reply.started":"2024-04-28T03:51:30.399062Z","shell.execute_reply":"2024-04-28T03:51:30.476737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate the corresponding model (must match the architecture of the state dict)\nb4_model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique(), pretrained=True).to(device)  # or your custom model class\n\n# Load the state dictionary into the model\nb4_model.load_state_dict(state_dict)","metadata":{"execution":{"iopub.status.busy":"2024-04-28T03:51:32.749889Z","iopub.execute_input":"2024-04-28T03:51:32.750722Z","iopub.status.idle":"2024-04-28T03:51:33.764451Z","shell.execute_reply.started":"2024-04-28T03:51:32.750677Z","shell.execute_reply":"2024-04-28T03:51:33.763540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prints out the summary of the model to ensure the model is loaded properly","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-04-28T03:51:38.545233Z","iopub.execute_input":"2024-04-28T03:51:38.545905Z","iopub.status.idle":"2024-04-28T03:51:51.533725Z","shell.execute_reply.started":"2024-04-28T03:51:38.545873Z","shell.execute_reply":"2024-04-28T03:51:51.532545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nimport torch\nimport torchvision.models as models  # Or import your model class\n\nsummary(b4_model, (3, 512, 512))","metadata":{"execution":{"iopub.status.busy":"2024-04-28T03:52:00.504374Z","iopub.execute_input":"2024-04-28T03:52:00.505268Z","iopub.status.idle":"2024-04-28T03:52:01.488670Z","shell.execute_reply.started":"2024-04-28T03:52:00.505230Z","shell.execute_reply":"2024-04-28T03:52:01.487723Z"},"trusted":true},"execution_count":null,"outputs":[]}]}