{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Train Inception-Resnet-V2 two class classification**","metadata":{}},{"cell_type":"markdown","source":"* [Dependencies and imports](#section-one)\n* [Basic configurations](#section-two)\n    - [Overide check_box function](#sub-section-one-one)\n    - [Check labels distribution](#sub-section-one-two)\n    - [Approaches for imbalanced data](#sub-section-one-three)\n* [Split data to folds](#section-three)\n* [Data augmentation using Albumentations](#section-four)\n* [Custom dataset](#section-five)\n* [Fitter](#section-six)\n* [Train](#section-seven)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"section-one\"></a>\n## **Dependencies and imports**","metadata":{}},{"cell_type":"code","source":"conda install gdcm -c conda-forge","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:47:30.261495Z","iopub.execute_input":"2021-09-22T13:47:30.261893Z","iopub.status.idle":"2021-09-22T13:48:31.800481Z","shell.execute_reply.started":"2021-09-22T13:47:30.261809Z","shell.execute_reply":"2021-09-22T13:48:31.799543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade --force-reinstall numpy","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:48:31.802248Z","iopub.execute_input":"2021-09-22T13:48:31.802644Z","iopub.status.idle":"2021-09-22T13:48:43.773048Z","shell.execute_reply.started":"2021-09-22T13:48:31.802584Z","shell.execute_reply":"2021-09-22T13:48:43.772123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:48:43.774939Z","iopub.execute_input":"2021-09-22T13:48:43.775203Z","iopub.status.idle":"2021-09-22T13:48:50.696820Z","shell.execute_reply.started":"2021-09-22T13:48:43.775176Z","shell.execute_reply":"2021-09-22T13:48:50.695802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchmetrics timm","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:48:50.700369Z","iopub.execute_input":"2021-09-22T13:48:50.700670Z","iopub.status.idle":"2021-09-22T13:48:56.845495Z","shell.execute_reply.started":"2021-09-22T13:48:50.700634Z","shell.execute_reply":"2021-09-22T13:48:56.844590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torch import nn\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nfrom glob import glob\nimport timm\nimport torchmetrics \nimport matplotlib.pyplot as plt\n# --- images --- \nimport cv2\nimport albumentations as A\n# --- time ---\nfrom datetime import datetime\nimport time\n# --- data ---\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom sklearn.model_selection import StratifiedShuffleSplit\n# --- wandb ---\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n# --- dicom ---\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-09-22T13:48:56.848711Z","iopub.execute_input":"2021-09-22T13:48:56.848975Z","iopub.status.idle":"2021-09-22T13:49:00.874318Z","shell.execute_reply.started":"2021-09-22T13:48:56.848947Z","shell.execute_reply":"2021-09-22T13:49:00.873420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OFFLINE = False\n\nif not OFFLINE:\n    user_secrets = UserSecretsClient()\n    wandb_key = user_secrets.get_secret(\"wandb-key\")\n    wandb.login(key=wandb_key)\n\n    run = wandb.init(project=\"siim-covid19-detection\", name=\"2-class-classification\", mode='online')","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:00.875834Z","iopub.execute_input":"2021-09-22T13:49:00.876190Z","iopub.status.idle":"2021-09-22T13:49:08.311671Z","shell.execute_reply.started":"2021-09-22T13:49:00.876154Z","shell.execute_reply":"2021-09-22T13:49:08.310851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-two\"></a>\n## **Basic configuration**","metadata":{}},{"cell_type":"code","source":"# --- configs ---\nNONE = 'none'\nOPACITY = 'opacity'\n\nclass Configs:\n    img_size = 1024\n    oversample = True\n    n_folds = 5\n    test_size = 0.15\n    classes = {NONE:0, OPACITY:1}\n    device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    batch_size = 4\n    num_workers = 8","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.314495Z","iopub.execute_input":"2021-09-22T13:49:08.314769Z","iopub.status.idle":"2021-09-22T13:49:08.344415Z","shell.execute_reply.started":"2021-09-22T13:49:08.314741Z","shell.execute_reply":"2021-09-22T13:49:08.343466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"sub-section-one-one\"></a>\n### **Check labels distribution**","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/d/miriamassraf/siim-covid19-detection/train_df.csv')\nfor cls in list(Configs.classes.keys()):\n    print(\"number of samples for class \\'{}\\': {}\".format(cls, len(train_df[train_df['image_level']==cls])))","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.347993Z","iopub.execute_input":"2021-09-22T13:49:08.348590Z","iopub.status.idle":"2021-09-22T13:49:08.423101Z","shell.execute_reply.started":"2021-09-22T13:49:08.348512Z","shell.execute_reply":"2021-09-22T13:49:08.422295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"sub-section-one-two\"></a>\n### **Approaches for imbalanced data**\n1. Oversample - \"create\" new data for the less common class </br>\n2. StratifiedShuffleSplit - balanced distribution of the data to folds </br>","metadata":{}},{"cell_type":"markdown","source":"<a id=\"section-three\"></a>\n## **Split data to folds**","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.424769Z","iopub.execute_input":"2021-09-22T13:49:08.425021Z","iopub.status.idle":"2021-09-22T13:49:08.446517Z","shell.execute_reply.started":"2021-09-22T13:49:08.424996Z","shell.execute_reply":"2021-09-22T13:49:08.445566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataFolds:\n    def __init__(self, train_df, continue_train=False):\n        assert Configs.n_folds > 0, \"num folds must be a positive number\"\n        if continue_train:\n            self.train_df = pd.read_csv('../input/d/miriamassraf/siim-covid19-detection/splitted_train_df.csv')\n        else:\n            self.train_df = train_df\n            if Configs.oversample:\n                # double the size of 'none' data (sample fraction of 1.0)\n                self.oversample('none', 1.0)\n\n            self.set_int_labels()\n            self.split_to_folds(Configs.test_size)\n    \n    def oversample(self, cls, frac):\n        rows_to_add = self.train_df[(self.train_df['image_level']==cls)&(self.train_df['study_level']=='negative')].sample(frac=frac, replace=True)\n        self.train_df = self.train_df.append(rows_to_add, ignore_index = True)\n        \n    def set_int_labels(self):\n        # set int labels for opacity and none\n        for index, row in self.train_df.iterrows():\n            if row['image_level'] == OPACITY:\n                self.train_df.loc[index, 'int_label'] = Configs.classes[OPACITY]\n            else:\n                self.train_df.loc[index, 'int_label'] = Configs.classes[NONE]\n        \n    def split_to_folds(self, test_size):\n        skf = StratifiedShuffleSplit(n_splits=Configs.n_folds, test_size=test_size)\n        for n, (train_index, val_index) in enumerate(skf.split(X=self.train_df.index, y=self.train_df['int_label'])):\n            self.train_df.loc[self.train_df.iloc[val_index].index, 'fold'] = int(n)\n        self.train_df = self.train_df[self.train_df['fold'].notna()]\n    \n    def get_train_df(self, fold_number): \n        if fold_number >= 0 and fold_number < Configs.n_folds:\n            return self.train_df[self.train_df['fold'] != fold_number]\n\n    def get_val_df(self, fold_number):\n        if fold_number >= 0 and fold_number < Configs.n_folds:\n            return self.train_df[self.train_df['fold'] == fold_number]","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.447878Z","iopub.execute_input":"2021-09-22T13:49:08.448214Z","iopub.status.idle":"2021-09-22T13:49:08.461913Z","shell.execute_reply.started":"2021-09-22T13:49:08.448179Z","shell.execute_reply":"2021-09-22T13:49:08.460956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualize distribution of labels over folds**","metadata":{}},{"cell_type":"code","source":"# Plot distibution\ndef plot_folds(data_folds):\n    nrows = Configs.n_folds//2\n    if Configs.n_folds%2 != 0:\n        nrows += 1\n    \n    fig, ax = plt.subplots(nrows=nrows, ncols=2, figsize=(20,10))\n    row = 0\n    for fold in range(Configs.n_folds):\n        if fold%2 == 0:\n            col = 0\n            if fold != 0:\n                row += 1\n        else:\n            col = 1\n\n        labels_count = {}\n        labels_count[OPACITY] = len(data_folds.train_df[((data_folds.train_df['fold'] == fold)&(data_folds.train_df['int_label'] == Configs.classes[OPACITY]))])\n        labels_count[NONE] = len(data_folds.train_df[((data_folds.train_df['fold'] == fold)&(data_folds.train_df['int_label'] == Configs.classes[NONE]))])\n        \n        ax[row, col].bar(list(labels_count.keys()), list(labels_count.values()))\n\n        for j, value in enumerate(labels_count.values()):\n            ax[row, col].text(j, value+2, str(value), color='#267DBE', fontweight='bold')\n\n        ax[row, col].grid(axis='y', alpha=0.75)\n        ax[row, col].set_title(\"For fold #{}\".format(fold), fontsize=15)\n        ax[row, col].set_ylabel(\"count\")","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.463426Z","iopub.execute_input":"2021-09-22T13:49:08.463851Z","iopub.status.idle":"2021-09-22T13:49:08.477572Z","shell.execute_reply.started":"2021-09-22T13:49:08.463815Z","shell.execute_reply":"2021-09-22T13:49:08.476768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folds = DataFolds(train_df)#, continue_train=True)\ndata_folds.train_df.to_csv(\"./splitted_train_df.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:08.478973Z","iopub.execute_input":"2021-09-22T13:49:08.479318Z","iopub.status.idle":"2021-09-22T13:49:11.352257Z","shell.execute_reply.started":"2021-09-22T13:49:08.479281Z","shell.execute_reply":"2021-09-22T13:49:11.351224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_folds(data_folds)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:11.353574Z","iopub.execute_input":"2021-09-22T13:49:11.353938Z","iopub.status.idle":"2021-09-22T13:49:12.018381Z","shell.execute_reply.started":"2021-09-22T13:49:11.353901Z","shell.execute_reply":"2021-09-22T13:49:12.017556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-four\"></a>\n## **Data augmentation using Albumentations**","metadata":{}},{"cell_type":"code","source":"def get_transforms(train=True):\n    if train:\n        return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.Rotate(limit=10),\n            A.OneOf([\n                A.Blur(blur_limit=3, p=0.5),\n                A.MedianBlur(blur_limit=3, p=0.5),\n                A.GaussNoise(p=0.5),\n                A.IAASharpen(p=0.5)\n                ],p=0.4),\n            A.CLAHE(p=0.6),\n            A.Resize(height=Configs.img_size, width=Configs.img_size, p=1),])\n\n    else:\n        return A.Compose([\n            A.Resize(height=Configs.img_size, width=Configs.img_size, p=1),])","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.019664Z","iopub.execute_input":"2021-09-22T13:49:12.019987Z","iopub.status.idle":"2021-09-22T13:49:12.027118Z","shell.execute_reply.started":"2021-09-22T13:49:12.019958Z","shell.execute_reply":"2021-09-22T13:49:12.026248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-five\"></a>\n## **Custom dataset**","metadata":{}},{"cell_type":"code","source":"def get_dicom_img(path):\n    data_file = pydicom.dcmread(path)\n    img = apply_voi_lut(data_file.pixel_array, data_file)\n\n    if data_file.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.amax(img) - img\n    \n    # Rescaling grey scale between 0-255 and convert to uint\n    img = img - np.min(img)\n    img = img / np.max(img)\n    img = (img * 255).astype(np.uint8)\n\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.028377Z","iopub.execute_input":"2021-09-22T13:49:12.028868Z","iopub.status.idle":"2021-09-22T13:49:12.042598Z","shell.execute_reply.started":"2021-09-22T13:49:12.028832Z","shell.execute_reply":"2021-09-22T13:49:12.041745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Covid19Dataset(Dataset):\n    def __init__(self, df, transform=None):\n        super().__init__()\n        self.df = df\n        self.transform = transform\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = row['dicom_path']\n        \n        img = get_dicom_img(img_path)\n        label = row['int_label']\n        \n        if self.transform:\n            transformed = self.transform(image=img)\n            img = transformed['image']\n           \n        # normalize img\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)\n        img /= 255.0\n        \n        # convert image into a torch.Tensor\n        img = torch.as_tensor(img, dtype=torch.float32)\n        #idx = torch.tensor([idx])\n        \n        # permute image to [C,H,W] from [H,W,C] and normalize\n        img = img.permute(2, 0, 1)\n        \n        return img, label\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.043962Z","iopub.execute_input":"2021-09-22T13:49:12.044392Z","iopub.status.idle":"2021-09-22T13:49:12.055566Z","shell.execute_reply.started":"2021-09-22T13:49:12.044357Z","shell.execute_reply":"2021-09-22T13:49:12.054773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset_fold(data_folds, fold, train=True):\n    if train:\n        return Covid19Dataset(data_folds.get_train_df(fold), transform=get_transforms(train))\n    return Covid19Dataset(data_folds.get_val_df(fold), transform=get_transforms(train))","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.056852Z","iopub.execute_input":"2021-09-22T13:49:12.057253Z","iopub.status.idle":"2021-09-22T13:49:12.070054Z","shell.execute_reply.started":"2021-09-22T13:49:12.057197Z","shell.execute_reply":"2021-09-22T13:49:12.068794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-six\"></a>\n## **Fitter**","metadata":{}},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.072594Z","iopub.execute_input":"2021-09-22T13:49:12.072940Z","iopub.status.idle":"2021-09-22T13:49:12.083924Z","shell.execute_reply.started":"2021-09-22T13:49:12.072907Z","shell.execute_reply":"2021-09-22T13:49:12.082962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Fitter:\n    def __init__(self, dir, model_name, verbose=True):\n        # create pretrained timm model by name\n        self.model_name = model_name\n        self.model = timm.create_model(model_name, pretrained=True, num_classes=len(Configs.classes))\n        self.verbose = verbose\n        \n        self.epoch = 0 \n        self.dir = dir\n        if not os.path.exists(self.dir):\n            os.makedirs(self.dir)\n        \n        self.log_path = os.path.join(self.dir, 'log.txt')\n        self.best_summary_loss = 10**5\n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=TrainConfigs.lr)\n        self.scheduler = TrainConfigs.SchedulerClass(self.optimizer, **TrainConfigs.scheduler_params) ########\n        self.log(f'Fitter prepared. Device is {Configs.device}')\n        \n    def fit(self, fold, train_loader, validation_loader):\n        self.model.to(Configs.device)\n        self.log(\"Fold {}\".format(fold))\n        for e in range(self.epoch, TrainConfigs.n_epochs):\n            if self.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n            \n            # train one epoch\n            t = time.time()\n            summary_loss, summary_accuracy = self.train_one_epoch(train_loader)\n            \n            # log train losses to console/log file\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch},\\ttotal loss: {summary_loss.avg:.5f},\\ttotal accuracy: {summary_accuracy.avg:.5f},\\ttime: {(time.time() - t):.5f}')\n            # log train losses to wandb\n            run.log({f\"{self.model_name}/train/total_loss_fold{fold}\": summary_loss.avg})\n\n            # validate one epoch\n            t = time.time()\n            summary_loss, summary_accuracy = self.validation_one_epoch(validation_loader)\n            \n            # log val losses to console/log file\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch},\\ttotal loss: {summary_loss.avg:.5f},\\ttotal accuracy: {summary_accuracy.avg:.5f},\\ttime: {(time.time() - t):.5f}')\n            # log val losses to wandb\n            run.log({f\"{self.model_name}/val/total_loss_fold{fold}\": summary_loss.avg})\n            \n            # save last checkpoint\n            self.save(os.path.join(self.dir, 'last-checkpoint.bin'))\n            wandb.save(os.path.join(self.dir, 'checkpoint-epoch{e}.bin'))\n            \n            # update best val losses and save best checkpoint if needed\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(os.path.join(self.dir, 'best-checkpoint.bin'))\n                wandb.save(os.path.join(self.dir, 'best-checkpoint.bin'))\n                for path in sorted(glob(os.path.join(self.dir, 'best-checkpoint.bin')))[:-3]:\n                    os.remove(path)\n\n            self.scheduler.step(metrics=summary_loss.avg) \n\n            self.epoch += 1\n                  \n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        summary_accuracy = AverageMeter()\n        \n        t = time.time()\n        for step, (images, labels) in enumerate(train_loader):\n            if self.verbose:\n                    print(f'Train Step {step}/{len(train_loader)},\\t' + \\\n                        f'total_loss: {summary_loss.avg:.5f},\\t' + \\\n                        f'total_accuracy: {summary_accuracy.avg:.5f},\\t' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            images = images.to(Configs.device).float()\n            labels = labels.to(Configs.device).long()\n            batch_size = images.shape[0]\n           \n            self.optimizer.zero_grad()\n            \n            logits = self.model(images)  \n            preds = logits.argmax(dim=1 , keepdim=True)\n            \n            loss = TrainConfigs.loss_fn(logits, labels)\n            accuracy = torchmetrics.functional.accuracy(labels, preds) ### labels, preds\n            \n            loss.backward()\n            self.optimizer.step()\n            \n            summary_loss.update(loss.detach().item(), batch_size)\n            summary_accuracy.update(accuracy, batch_size)\n            \n            del images, labels\n            torch.cuda.empty_cache()\n\n        return summary_loss, summary_accuracy\n    \n    def validation_one_epoch(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        summary_accuracy = AverageMeter()\n        \n        t = time.time()\n        for step, (images, labels) in enumerate(val_loader):\n            if self.verbose:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'total_loss: {summary_loss.avg:.5f}, ' + \\\n                        f'total_accuracy: {summary_accuracy.avg:.5f},\\t' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                images = images.to(Configs.device).float()\n                labels = labels.to(Configs.device).long()\n                batch_size = images.shape[0]\n    \n                logits = self.model(images)\n                preds = logits.argmax(dim=1, keepdim=True)\n            \n                loss = TrainConfigs.loss_fn(logits, labels)\n                accuracy = torchmetrics.functional.accuracy(labels, preds)\n                \n                summary_loss.update(loss.detach().item(), batch_size)\n                summary_accuracy.update(accuracy, batch_size)\n                \n            del images, labels\n            torch.cuda.empty_cache()\n            \n        return summary_loss, summary_accuracy\n    \n    # save checkpoint to path\n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    # load checkpoint from path\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n    \n    # log to console/log file\n    def log(self, message):\n        if self.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.085460Z","iopub.execute_input":"2021-09-22T13:49:12.085883Z","iopub.status.idle":"2021-09-22T13:49:12.130434Z","shell.execute_reply.started":"2021-09-22T13:49:12.085848Z","shell.execute_reply":"2021-09-22T13:49:12.129505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"section-seven\"></a>\n## **Train**","metadata":{}},{"cell_type":"markdown","source":"**Train configurations**","metadata":{}},{"cell_type":"code","source":"class TrainConfigs:\n    n_epochs = 10\n    lr = 0.001\n    loss_fn = nn.CrossEntropyLoss() \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=2,\n        verbose=True, \n        threshold=0.0001,\n        threshold_mode='abs',\n        min_lr=1e-8,\n    )","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.131749Z","iopub.execute_input":"2021-09-22T13:49:12.132229Z","iopub.status.idle":"2021-09-22T13:49:12.140447Z","shell.execute_reply.started":"2021-09-22T13:49:12.132188Z","shell.execute_reply":"2021-09-22T13:49:12.139694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Run train** ","metadata":{}},{"cell_type":"code","source":"def run_training(model_name, fold, train_dataset, val_dataset):\n    # create train/validation data loaders\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=Configs.batch_size,\n        sampler=RandomSampler(train_dataset),\n        pin_memory=False,\n        drop_last=True,\n        num_workers=Configs.num_workers,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        val_dataset, \n        batch_size=Configs.batch_size,\n        num_workers=Configs.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(val_dataset),\n        pin_memory=False,\n    )\n    \n    # create and run fitter for model\n    fitter = Fitter(f'./{model_name}/{model_name}_fold{fold}', model_name)\n    fitter.fit(fold, train_loader, val_loader)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.141696Z","iopub.execute_input":"2021-09-22T13:49:12.142072Z","iopub.status.idle":"2021-09-22T13:49:12.150338Z","shell.execute_reply.started":"2021-09-22T13:49:12.142037Z","shell.execute_reply":"2021-09-22T13:49:12.149418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Run train for 5 models over the different folds**","metadata":{}},{"cell_type":"code","source":"# wanted to try more models but eventually use only one - inception_resnet_v2\nmodels = ['inception_resnet_v2', 'pnasnet5large', 'inception_v4']","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.151659Z","iopub.execute_input":"2021-09-22T13:49:12.152078Z","iopub.status.idle":"2021-09-22T13:49:12.158587Z","shell.execute_reply.started":"2021-09-22T13:49:12.152043Z","shell.execute_reply":"2021-09-22T13:49:12.157692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\ntrain_dataset = get_dataset_fold(data_folds, fold)\nval_dataset = get_dataset_fold(data_folds, fold, train=False)\n\nrun_training(models[0], fold, train_dataset, val_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T13:49:12.162068Z","iopub.execute_input":"2021-09-22T13:49:12.162379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(Configs.n_folds):\n    train_dataset = get_dataset_fold(data_folds, fold)\n    val_dataset = get_dataset_fold(data_folds, fold, train=False)\n\n    run_training(models[0], fold, train_dataset, val_dataset)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**zip results and save files**","metadata":{}},{"cell_type":"code","source":"!zip -r ./inception_resnet_v2.zip ./inception_resnet_v2","metadata":{"execution":{"iopub.status.busy":"2021-07-29T12:44:31.535653Z","iopub.status.idle":"2021-07-29T12:44:31.536056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('./inception_resnet_v2.zip')","metadata":{"execution":{"iopub.status.busy":"2021-07-29T12:44:31.538105Z","iopub.status.idle":"2021-07-29T12:44:31.538842Z"},"trusted":true},"execution_count":null,"outputs":[]}]}