{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Histopathologic Notebook\n","metadata":{}},{"cell_type":"markdown","source":"## 1. Dependencies\n","metadata":{}},{"cell_type":"code","source":"import sys\n\nsys.modules[\"IPython\"] = None\nsys.stderr = sys.stdout\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport os\n\nos.environ[\"PYTHONWARNINGS\"] = \"ignore\"","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:24.070358Z","iopub.status.busy":"2024-10-11T11:30:24.069842Z","iopub.status.idle":"2024-10-11T11:30:24.080831Z","shell.execute_reply":"2024-10-11T11:30:24.079982Z","shell.execute_reply.started":"2024-10-11T11:30:24.070321Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    from captum.attr import IntegratedGradients\nexcept:\n    !pip3 install captum==0.7.0","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchvision import models\nfrom torchvision.transforms import v2 as transforms\n\nimport multiprocessing as mp\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, roc_auc_score, RocCurveDisplay\nfrom sklearn.linear_model import LogisticRegression\n\nfrom optuna import logging, create_study\nfrom optuna.visualization.matplotlib import plot_optimization_history\nfrom optuna.exceptions import TrialPruned\n\nlogging.set_verbosity(logging.WARNING)\n\nfrom captum.attr import IntegratedGradients\n\nimport numpy as np\nimport pandas as pd\nimport tifffile as tiff\n\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport textwrap\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:24.096365Z","iopub.status.busy":"2024-10-11T11:30:24.096072Z","iopub.status.idle":"2024-10-11T11:30:27.702405Z","shell.execute_reply":"2024-10-11T11:30:27.701619Z","shell.execute_reply.started":"2024-10-11T11:30:24.096332Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Data exploration\n","metadata":{}},{"cell_type":"markdown","source":"First we will create a class that allows us to easily read the provided data.\n","metadata":{}},{"cell_type":"code","source":"class TifImageDataset(Dataset):\n\n    def __init__(self, imgs_dir, labels_file=None, transforms=None):\n        self.imgs_dir = imgs_dir\n        if labels_file is None:\n            self.ids = np.array([f.split(\".\")[0] for f in os.listdir(imgs_dir)])\n            self.labels = None\n        else:\n            df = pd.read_csv(labels_file)\n            self.ids = df[\"id\"].to_numpy()\n            self.labels = df[\"label\"].to_numpy()\n        self.transforms = transforms\n\n    def __len__(self):\n        return self.ids.shape[0]\n\n    def __getitem__(self, idx):\n        image = tiff.imread(os.path.join(self.imgs_dir, self.ids[idx] + \".tif\"))\n        if self.transforms is not None:\n            image = self.transforms(image)\n        return (image,) if self.labels is None else (image, self.labels[idx])","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:27.705840Z","iopub.status.busy":"2024-10-11T11:30:27.705033Z","iopub.status.idle":"2024-10-11T11:30:27.712509Z","shell.execute_reply":"2024-10-11T11:30:27.711536Z","shell.execute_reply.started":"2024-10-11T11:30:27.705795Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's take a look at some of the images along with their classification. For this, we will also use a function that shows us the image in question.\n","metadata":{}},{"cell_type":"code","source":"def imshow(imgs, titles, max_cols=5):\n    n_imgs = len(imgs)\n    n_rows = int(np.ceil(n_imgs / max_cols))\n    n_cols = n_imgs if n_imgs <= max_cols else max_cols\n\n    _, axes = plt.subplots(nrows=n_rows, ncols=n_cols, squeeze=False)\n\n    for i in range(n_rows):\n        for j in range(n_cols):\n            if j + n_cols * i < n_imgs:\n                axes[i][j].imshow(imgs[j + n_cols * i])\n                axes[i][j].set_title(f\"{titles[j + n_cols * i]}\")\n                axes[i][j].set_xticks([])\n                axes[i][j].set_yticks([])\n            else:\n                axes[i][j].axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imgs_dir = \"../input/histopathologic-cancer-detection/train\"\nlabels_file = \"../input/histopathologic-cancer-detection/train_labels.csv\"\n\norig_dataset = TifImageDataset(imgs_dir, labels_file)\n\nimgs, titles = [], []\nfor idx in range(5, 20):\n    img, label = orig_dataset[idx]\n    imgs.append(img)\n    titles.append(f\"Label: {label}\")\n\nimshow(imgs, titles)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Apparently the number of positives as well as negatives is the same, let's check it.\n","metadata":{}},{"cell_type":"code","source":"index, values = np.unique(orig_dataset.labels, return_counts=True)\n\nprint(f\"Label 0: {100 * values[0] / len(orig_dataset):.1f}%\")\nprint(f\"Label 1: {100 * values[1] / len(orig_dataset):.1f}%\")\n\nplt.figure(figsize=(6, 6))\nplt.bar(index, values, color=[\"purple\", \"orange\"])\nplt.xticks(index)\nplt.yticks(values)\nplt.title(\"Histopathologic cancer distribution\")\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As it is not exactly distributed, it will be good to take it into account when training and validating.\n","metadata":{}},{"cell_type":"markdown","source":"## 3. Previous preparation\n","metadata":{}},{"cell_type":"markdown","source":"Before we go through the different techniques used to try to achieve the best percentage, we will introduce auxiliary classes, as well as parameters, common to all of them.\n\n\n\nFirst it will be a class that will help us to train, evaluate and predict. Keep in mind that it will be as generic as possible to be useful in all cases.\n","metadata":{}},{"cell_type":"code","source":"class NeuralNetworkTrainer:\n\n    def __init__(\n        self, model, loss_f, optimizer, scheduler=None, device=torch.device(\"cpu\")\n    ):\n        self.model = model\n        self.loss_f = loss_f\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.device = device\n\n        self.mtr_ = None\n        self.loss_ = None\n        self.vmtr_ = None\n        self.vloss_ = None\n        self.history_ = {\"mtr\": [], \"loss\": [], \"vmtr\": [], \"vloss\": []}\n\n        self.__probs_metrics = [\"roc_auc_score\"]\n\n    def execute(\n        self,\n        dataloader,\n        vdataloader=None,\n        metric=accuracy_score,\n        pv_class=-1,\n        epochs=5,\n        early_stopper=None,\n        verbose=1,\n        trial=None,\n    ):\n        num_batch = len(dataloader)\n\n        progress_bar = None\n\n        for e in range(1, 1 + epochs):\n\n            if verbose > 0:\n                progress_bar = tqdm(total=num_batch, unit=\"batch\", desc=f\"Epoch {e}\")\n\n            self.__train_one_epoch(dataloader, progress_bar)\n            self.mtr_, self.loss_ = self.eval(dataloader, metric, True, pv_class)\n            self.history_[\"mtr\"].append(self.mtr_)\n            self.history_[\"loss\"].append(self.loss_)\n\n            if verbose > 0:\n                post = {\"mtr\": f\"{self.mtr_:.4f}\", \"loss\": f\"{self.loss_:.4f}\"}\n                progress_bar.set_postfix(**post)\n\n            if vdataloader:\n                self.vmtr_, self.vloss_ = self.eval(vdataloader, metric, True, pv_class)\n                self.history_[\"vmtr\"].append(self.vmtr_)\n                self.history_[\"vloss\"].append(self.vloss_)\n\n                if verbose > 0:\n                    vpost = {\"vmtr\": f\"{self.vmtr_:.4f}\", \"vloss\": f\"{self.vloss_:.4f}\"}\n                    progress_bar.set_postfix(**post, **vpost)\n\n                if early_stopper(self.vloss_):\n                    if trial is not None:\n                        trial.report(self.vmtr_, e)\n                    if verbose > -1:\n                        if verbose > 0:\n                            progress_bar.close()\n                        print(f\"Early stop at epoch {e}\")\n                    break\n\n            if self.scheduler is not None:\n                self.scheduler.step()\n\n            if verbose > 0:\n                progress_bar.close()\n\n            if trial is not None:\n                trial.report(self.vmtr_ if vdataloader else self.mtr_, e)\n                if trial.should_prune():\n                    raise TrialPruned()\n\n        if verbose > -1:\n            print(f\"Training: mtr={self.mtr_:.4f}, loss={self.loss_:.4f}\")\n            if vdataloader:\n                print(f\"Evaluation: vmtr={self.vmtr_:.4f}, vloss={self.vloss_:.4f}\")\n\n        if verbose == 2:\n            executed_epochs = range(1, e + 1)\n            plt.figure(figsize=(6, 6))\n            plt.plot(executed_epochs, self.history_[\"mtr\"], label=\"mtr\")\n            plt.plot(executed_epochs, self.history_[\"loss\"], label=\"loss\")\n            if vdataloader:\n                plt.plot(executed_epochs, self.history_[\"vmtr\"], label=\"vmtr\")\n                plt.plot(executed_epochs, self.history_[\"vloss\"], label=\"vloss\")\n            plt.title(\"Metric & Loss History\")\n            plt.legend()\n            plt.show()\n\n        return self\n\n    def __train_one_epoch(self, dataloader, progress_bar):\n        self.model.train()\n\n        for data in dataloader:\n            data = list(map(lambda x: x.to(self.device), data))\n\n            Xs, y = data[:-1], data[-1]\n\n            pred = self.model(*Xs)\n            loss = self.loss_f(pred, y)\n\n            loss.backward()\n            self.optimizer.step()\n            self.optimizer.zero_grad()\n\n            if progress_bar:\n                progress_bar.update(1)\n\n    def eval(self, dataloader, metric=accuracy_score, calc_loss=False, pv_class=-1):\n        calc_probs = True if metric.__name__ in self.__probs_metrics else False\n        data = self.predict(dataloader, True, calc_loss, calc_probs)\n\n        y_pred = data[0] if not calc_probs else data[0][:, pv_class]\n        y_true = data[1]\n        loss = data[2] if calc_loss else None\n\n        score = metric(y_true, y_pred)\n\n        res = []\n        res.append(score)\n        if calc_loss:\n            res.append(loss)\n\n        return (*res,)\n\n    def predict(self, dataloader, has_target=False, calc_loss=False, calc_probs=False):\n        total_loss = 0.0\n        all_ys = []\n        all_preds = []\n\n        num_batch = len(dataloader)\n\n        self.model.eval()\n\n        with torch.no_grad():\n            for data in dataloader:\n                data = list(map(lambda x: x.to(self.device), data))\n                if has_target:\n                    Xs, y = data[:-1], data[-1]\n                else:\n                    Xs = data\n\n                preds = self.model(*Xs)\n                if calc_probs:\n                    all_preds.extend(F.softmax(preds, dim=1).cpu().numpy())\n                else:\n                    all_preds.extend(preds.argmax(dim=1).cpu().numpy())\n\n                if has_target:\n                    if calc_loss:\n                        total_loss += self.loss_f(preds, y).item()\n                    all_ys.extend(y.cpu().numpy())\n\n        res = []\n        res.append(np.array(all_preds))\n        if has_target:\n            res.append(np.array(all_ys))\n            if calc_loss:\n                res.append(total_loss / num_batch)\n\n        return (*res,)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Also for the realization of the early stop and to avoid overfitting, we will make another auxiliary class to the previous one.\n","metadata":{}},{"cell_type":"code","source":"class EarlyStopper:\n\n    def __init__(self, patience=1, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n\n        self.counter = 0\n        self.min_vloss = np.inf\n\n    def __call__(self, vloss):\n        if vloss < self.min_vloss:\n            self.counter = 0\n            self.min_vloss = vloss\n\n        elif vloss > (self.min_vloss + self.min_delta):\n            self.counter += 1\n            if self.counter >= self.patience:\n                return True\n\n        return False","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We will set the GPU as a device, in addition to some hyperparameters that are common (they have been tuned during the creation of the notebook and the same ones have been reached for all cases).\n","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nbatch_size = 256\nepochs = 15\nlearning_rate = 0.001\nearly_stopper_args = {\"patience\": 2, \"min_delta\": 0.001}","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We will then read the data but indicating transformations to avoid overfitting in the training set. Note that this is very important since Whole Slide Image has been performed so, even if we split the set in training and validation, data leakage will occur and the performance obtained in validation may vary a lot with respect to the test set. For this reason, the set will be split only once instead of cross-validation. It should also be noted that we have simply applied the transformations without adding new images since the final performance obtained is quite close to 1, the main problem being overfitting, so we save a little computational cost.\n","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose(\n    [\n        transforms.ToTensor(),\n        transforms.RandomChoice(\n            [\n                transforms.ColorJitter(brightness=0.5),\n                transforms.ColorJitter(contrast=0.5),\n                transforms.ColorJitter(saturation=0.5),\n                transforms.ColorJitter(hue=0.5),\n                transforms.ColorJitter(0.1, 0.1, 0.1, 0.1),\n                transforms.ColorJitter(0.3, 0.3, 0.3, 0.3),\n                transforms.ColorJitter(0.5, 0.5, 0.5, 0.5),\n            ]\n        ),\n        transforms.RandomChoice(\n            [\n                transforms.RandomRotation((0, 0)),\n                transforms.RandomHorizontalFlip(p=1),\n                transforms.RandomVerticalFlip(p=1),\n                transforms.RandomRotation((90, 90)),\n                transforms.RandomRotation((180, 180)),\n                transforms.RandomRotation((270, 270)),\n                transforms.Compose(\n                    [\n                        transforms.RandomHorizontalFlip(p=1),\n                        transforms.RandomRotation((90, 90)),\n                    ]\n                ),\n                transforms.Compose(\n                    [\n                        transforms.RandomHorizontalFlip(p=1),\n                        transforms.RandomRotation((270, 270)),\n                    ]\n                ),\n            ]\n        ),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\n\ndataset = TifImageDataset(imgs_dir, labels_file, transform)\n\ntrain_idx, val_idx = train_test_split(\n    np.arange(len(dataset)),\n    test_size=0.25,\n    shuffle=True,\n    stratify=dataset.labels,\n)\n\ntrain_dataset = Subset(dataset, train_idx)\nval_dataset = Subset(dataset, val_idx)\n\ntrain_dataloader = DataLoader(\n    train_dataset,\n    batch_size,\n    shuffle=True,\n    num_workers=mp.cpu_count(),\n    pin_memory=torch.cuda.is_available(),\n)\nval_dataloader = DataLoader(\n    val_dataset,\n    batch_size,\n    shuffle=False,\n    num_workers=mp.cpu_count(),\n    pin_memory=torch.cuda.is_available(),\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In addition, for the tests performed before the final model, we will train it with part of the total set, to speed up the process.\n","metadata":{}},{"cell_type":"code","source":"subset_size = 10000\n\n_, train_idx = train_test_split(\n    np.arange(len(dataset)),\n    test_size=subset_size / len(dataset),\n    shuffle=True,\n    stratify=dataset.labels,\n)\n_, val_idx = train_test_split(\n    np.delete(np.arange(len(dataset)), train_idx),\n    test_size=0.25 * subset_size / (len(dataset) - subset_size),\n    shuffle=True,\n    stratify=np.delete(dataset.labels, train_idx),\n)\n\ntrain_subset = Subset(dataset, train_idx)\nval_subset = Subset(dataset, val_idx)\n\ntrain_subloader = DataLoader(\n    train_subset,\n    batch_size,\n    shuffle=True,\n    num_workers=mp.cpu_count(),\n    pin_memory=torch.cuda.is_available(),\n)\nval_subloader = DataLoader(\n    val_subset,\n    batch_size,\n    shuffle=False,\n    num_workers=mp.cpu_count(),\n    pin_memory=torch.cuda.is_available(),\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Approaches\n","metadata":{}},{"cell_type":"markdown","source":"## &#160;&#160;&#160;&#160;4.1. Convolutional Neural Network\n","metadata":{}},{"cell_type":"markdown","source":"First we will see a convolutional neural network created and customized from scratch. To do this, we will first create an auxiliary class that will allow us to define a global max pooling that will be used at the end of the convolutional blocks, as well as another class to define these blocks, which will be of the form conv -> relu -> pool -> bn.\n","metadata":{}},{"cell_type":"code","source":"class GlobalMaxPooling(nn.Module):\n\n    def __init__(self, *args, **kwargs):\n        super().__init__()\n\n    def forward(self, x):\n        return torch.amax(x, dim=(2, 3))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Convolutional2dBlock(nn.Module):\n\n    def __init__(\n        self,\n        in_ch,\n        out_ch,\n        kernel,\n        stride=1,\n        padding=0,\n        batch_norm=False,\n        pool_type=None,\n        pool_size=2,\n        pool_stride=1,\n        pool_padding=0,\n    ):\n        super().__init__()\n\n        self.cv = nn.Conv2d(in_ch, out_ch, kernel, stride, padding, bias=not batch_norm)\n\n        self.bn = None\n        if batch_norm:\n            if \"global\" in pool_type.__name__.lower():\n                self.bn = nn.BatchNorm1d(out_ch)\n            else:\n                self.bn = nn.BatchNorm2d(out_ch)\n\n        self.pl = None\n        if pool_type:\n            self.pl = pool_type(pool_size, pool_stride, pool_padding)\n\n    def forward(self, x):\n        x = F.relu(self.cv(x))\n        if self.pl is not None:\n            x = self.pl(x)\n        if self.bn is not None:\n            x = self.bn(x)\n        return x","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Obviously, these two auxiliary classes will be generic, but so will be the CNN that we will define below. This is to make it easier to also do a tuning of the network architecture that we will see later.\n\n\n\nIn general, the proposed architecture will consist of convolutional blocks, taking into account that the last one will have a global pooling, followed by the fc layers.\n","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n\n    def __init__(\n        self,\n        in_chl,\n        out_size,\n        cv_args=[(16, 3, 1, 0), (8, 3, 1, 0)],\n        bt_norm=[False, False],\n        pl_args=[(nn.MaxPool2d, 2, 1, 0), (GlobalMaxPooling,)],\n        fc_args=[(8, 0.2)],\n    ):\n        super().__init__()\n\n        in_chls = [in_chl] + list(map(lambda cv_arg: cv_arg[0], cv_args[:-1]))\n        self.conv_blocks = nn.Sequential(\n            *[\n                Convolutional2dBlock(in_chls[i], *cv_args[i], bt_norm[i], *pl_args[i])\n                for i in torch.arange(len(cv_args))\n            ]\n        )\n\n        flatten = lambda xss: sum(xss, [])\n        self.fc = (\n            nn.Linear(cv_args[-1][0], out_size)\n            if len(fc_args) == 0\n            else nn.Sequential(\n                nn.Linear(cv_args[-1][0], fc_args[0][0]),\n                nn.ReLU(),\n                nn.Dropout(fc_args[0][1]),\n                *flatten(\n                    [\n                        [\n                            nn.Linear(fc_args[i - 1][0], fc_args[i][0]),\n                            nn.ReLU(),\n                            nn.Dropout(fc_args[i][1]),\n                        ]\n                        for i in torch.arange(1, len(fc_args))\n                    ]\n                ),\n                nn.Linear(fc_args[-1][0], out_size),\n            )\n        )\n\n    def forward(self, x):\n        x = self.conv_blocks(x)\n        x = self.fc(x)\n        return x","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:27.714518Z","iopub.status.busy":"2024-10-11T11:30:27.713966Z","iopub.status.idle":"2024-10-11T11:30:27.730560Z","shell.execute_reply":"2024-10-11T11:30:27.729675Z","shell.execute_reply.started":"2024-10-11T11:30:27.714479Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's run a simple version of the architecture without worrying too much about tuning to see the performance a priori.\n","metadata":{}},{"cell_type":"code","source":"model = CNN(\n    in_chl=3,\n    out_size=2,\n    cv_args=[(8, 2, 1, 1), (16, 2, 1, 1)],\n    pl_args=[(nn.MaxPool2d, 2, 2, 0), (GlobalMaxPooling,)],\n    bt_norm=[True, True],\n    fc_args=[(8, 0.2)],\n).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.7**epoch)\n\nnet = NeuralNetworkTrainer(\n    model,\n    criterion,\n    optimizer,\n    scheduler=scheduler,\n    device=device,\n).execute(\n    train_subloader,\n    vdataloader=val_subloader,\n    metric=roc_auc_score,\n    epochs=epochs,\n    early_stopper=EarlyStopper(**early_stopper_args),\n    verbose=2,\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:27.778601Z","iopub.status.busy":"2024-10-11T11:30:27.778185Z","iopub.status.idle":"2024-10-11T11:30:28.158980Z","shell.execute_reply":"2024-10-11T11:30:28.158126Z","shell.execute_reply.started":"2024-10-11T11:30:27.778541Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"In the following we will try to tune some parts of the architecture, but not all, as some parts should work better as indicated below. Note that the number of epochs has been reduced to speed up the process.\n","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    n_conv_blocks = trial.suggest_int(\"n_conv_blocks\", 2, 3)\n\n    cv_args = []\n    bt_norm = []\n    pl_args = []\n\n    for i in range(n_conv_blocks):\n\n        output_channel = trial.suggest_int(f\"output_channel_{i}\", 16, 128, step=16)\n        kernel_size = trial.suggest_int(f\"kernel_size_{i}\", 3, 5, step=2)\n        stride = 1\n        padding = \"same\"\n        cv_args.append((output_channel, kernel_size, stride, padding))\n\n        batch_norm = trial.suggest_categorical(f\"batch_norm_{i}\", [False, True])\n        bt_norm.append(batch_norm)\n\n        pool_type = nn.MaxPool2d if i < n_conv_blocks - 1 else GlobalMaxPooling\n        pool_kernel_size = 2\n        pool_stride = trial.suggest_int(f\"pool_stride_{i}\", 1, 2)\n        pool_padding = 0\n        pl_args.append((pool_type, pool_kernel_size, pool_stride, pool_padding))\n\n    n_fc_layers = trial.suggest_int(\"n_fc_layers\", 1, 3)\n\n    fc_args = []\n\n    for i in range(n_fc_layers):\n\n        middle_size = trial.suggest_int(f\"middle_size_{i}\", 4, 128, step=4)\n        dropout_rate = trial.suggest_float(f\"dropout_rate_{i}\", 0.2, 0.5)\n        fc_args.append((middle_size, dropout_rate))\n\n    model = CNN(\n        in_chl=3,\n        out_size=2,\n        cv_args=cv_args,\n        bt_norm=bt_norm,\n        pl_args=pl_args,\n        fc_args=fc_args,\n    ).to(device)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda ep: 0.7**ep)\n\n    net = NeuralNetworkTrainer(\n        model,\n        criterion,\n        optimizer,\n        scheduler=scheduler,\n        device=device,\n    ).execute(\n        train_subloader,\n        vdataloader=val_subloader,\n        metric=roc_auc_score,\n        epochs=2,\n        early_stopper=EarlyStopper(**early_stopper_args),\n        verbose=-1,\n        trial=trial,\n    )\n\n    return net.vmtr_","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.365157Z","iopub.status.busy":"2024-10-11T11:30:28.364852Z","iopub.status.idle":"2024-10-11T11:30:28.380422Z","shell.execute_reply":"2024-10-11T11:30:28.379592Z","shell.execute_reply.started":"2024-10-11T11:30:28.365111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=10, show_progress_bar=True)\n\noutput = \"CNN: %.3f with %s\" % (\n    study.best_trial.values[0],\n    study.best_trial.params,\n)\nprint(\"\\n\".join(textwrap.wrap(output, 88, subsequent_indent=\"\\t\")))\nplot_optimization_history(study, target_name=\"Performance\")\nplt.title(\"CNN\")\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.381823Z","iopub.status.busy":"2024-10-11T11:30:28.381484Z","iopub.status.idle":"2024-10-11T11:30:28.395806Z","shell.execute_reply":"2024-10-11T11:30:28.394965Z","shell.execute_reply.started":"2024-10-11T11:30:28.381790Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The results obtained are (taking into account that the randomness has not been fixed):\n\n\n\n- CNN: 0.843 with {'n_conv_blocks': 3, 'output_channel_0': 96, 'kernel_size_0': 5, 'batch_norm_0': True, 'pool_stride_0': 2, 'output_channel_1': 112, 'kernel_size_1': 3, 'batch_norm_1': True, 'pool_stride_1': 1, 'output_channel_2': 128, 'kernel_size_2': 3, 'batch_norm_2': False, 'pool_stride_2': 1, 'n_fc_layers': 3, 'middle_size_0': 96, 'dropout_rate_0': 0.44303403757732845, 'middle_size_1': 80, 'dropout_rate_1': 0.36195297000746773, 'middle_size_2': 64, 'dropout_rate_2': 0.29013193947373683}\n\n\n\nLet's see how it performs with all epochs (also, we go back to the previous matplotlib settings since optuna modifies them).\n","metadata":{}},{"cell_type":"code","source":"mpl.rcParams.update(mpl.rcParamsDefault)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.397266Z","iopub.status.busy":"2024-10-11T11:30:28.396976Z","iopub.status.idle":"2024-10-11T11:30:28.411469Z","shell.execute_reply":"2024-10-11T11:30:28.410628Z","shell.execute_reply.started":"2024-10-11T11:30:28.397235Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = CNN(\n    in_chl=3,\n    out_size=2,\n    cv_args=[\n        (96, 5, 1, \"same\"),\n        (112, 3, 1, \"same\"),\n        (128, 3, 1, \"same\"),\n    ],\n    bt_norm=[True, True, False],\n    pl_args=[\n        (nn.MaxPool2d, 2, 2, 0),\n        (nn.MaxPool2d, 2, 1, 0),\n        (GlobalMaxPooling,),\n    ],\n    fc_args=[(96, 0.45), (80, 0.35), (64, 0.3)],\n).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.7**epoch)\n\ncnn = NeuralNetworkTrainer(\n    model,\n    criterion,\n    optimizer,\n    scheduler=scheduler,\n    device=device,\n).execute(\n    train_subloader,\n    vdataloader=val_subloader,\n    metric=roc_auc_score,\n    epochs=epochs,\n    early_stopper=EarlyStopper(**early_stopper_args),\n    verbose=2,\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.412896Z","iopub.status.busy":"2024-10-11T11:30:28.412591Z","iopub.status.idle":"2024-10-11T11:30:28.428537Z","shell.execute_reply":"2024-10-11T11:30:28.427708Z","shell.execute_reply.started":"2024-10-11T11:30:28.412863Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## &#160;&#160;&#160;&#160;4.2. ResNet18\n","metadata":{}},{"cell_type":"markdown","source":"Next we will test some pre-trained models. The first one will be ResNet. The version with the best results, i.e. ResNet18, was selected. Note that only the fc layers will be trained to avoid overfitting.\n","metadata":{}},{"cell_type":"code","source":"model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)\nfor param in model.parameters():\n    param.requires_grad = False\nmodel.fc = nn.Linear(model.fc.in_features, 2)\nmodel.to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.7**epoch)\n\nresnet = NeuralNetworkTrainer(\n    model,\n    criterion,\n    optimizer,\n    scheduler=scheduler,\n    device=device,\n).execute(\n    train_subloader,\n    vdataloader=val_subloader,\n    metric=roc_auc_score,\n    epochs=epochs,\n    early_stopper=EarlyStopper(**early_stopper_args),\n    verbose=2,\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.430175Z","iopub.status.busy":"2024-10-11T11:30:28.429863Z","iopub.status.idle":"2024-10-11T11:30:28.700667Z","shell.execute_reply":"2024-10-11T11:30:28.699374Z","shell.execute_reply.started":"2024-10-11T11:30:28.430142Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## &#160;&#160;&#160;&#160;4.3. VGG16\n","metadata":{}},{"cell_type":"markdown","source":"The next one will be VGG, which in this case, the version chosen, is VGG16.\n","metadata":{}},{"cell_type":"code","source":"model = models.vgg16(weights=models.VGG16_Weights.DEFAULT)\nfor param in model.parameters():\n    param.requires_grad = False\nfor param in model.classifier.parameters():\n    param.requires_grad = True\nmodel.classifier[-1] = nn.Linear(model.classifier[-1].in_features, 2)\nmodel.to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.7**epoch)\n\nvgg = NeuralNetworkTrainer(\n    model,\n    criterion,\n    optimizer,\n    scheduler=scheduler,\n    device=device,\n).execute(\n    train_subloader,\n    vdataloader=val_subloader,\n    metric=roc_auc_score,\n    epochs=epochs,\n    early_stopper=EarlyStopper(**early_stopper_args),\n    verbose=2,\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:28.702916Z","iopub.status.busy":"2024-10-11T11:30:28.702452Z","iopub.status.idle":"2024-10-11T11:30:30.331583Z","shell.execute_reply":"2024-10-11T11:30:30.330516Z","shell.execute_reply.started":"2024-10-11T11:30:28.702862Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## &#160;&#160;&#160;&#160;4.4. Stacking ensemble\n","metadata":{}},{"cell_type":"markdown","source":"Finally, we will test how the three previous models work in an ensemble, specifically, stacking ensemble.\n","metadata":{}},{"cell_type":"code","source":"pred_train_cnn, y_train = cnn.predict(train_subloader, has_target=True)\npred_train_res, y_train = resnet.predict(train_subloader, has_target=True)\npred_train_vgg, y_train = vgg.predict(train_subloader, has_target=True)\n\npred_val_cnn, y_val = cnn.predict(val_subloader, has_target=True)\npred_val_res, y_val = resnet.predict(val_subloader, has_target=True)\npred_val_vgg, y_val = vgg.predict(val_subloader, has_target=True)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:30.333155Z","iopub.status.busy":"2024-10-11T11:30:30.332843Z","iopub.status.idle":"2024-10-11T11:30:30.337720Z","shell.execute_reply":"2024-10-11T11:30:30.336625Z","shell.execute_reply.started":"2024-10-11T11:30:30.333121Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stacked_train_preds = np.vstack((pred_train_cnn, pred_train_res, pred_train_vgg))\nstacked_val_preds = np.vstack((pred_val_cnn, pred_val_res, pred_val_vgg))","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:30.339065Z","iopub.status.busy":"2024-10-11T11:30:30.338780Z","iopub.status.idle":"2024-10-11T11:30:30.353090Z","shell.execute_reply":"2024-10-11T11:30:30.352191Z","shell.execute_reply.started":"2024-10-11T11:30:30.339034Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_learner = LogisticRegression().fit(stacked_train_preds.T, y_train)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:30.354860Z","iopub.status.busy":"2024-10-11T11:30:30.354256Z","iopub.status.idle":"2024-10-11T11:30:30.364397Z","shell.execute_reply":"2024-10-11T11:30:30.363487Z","shell.execute_reply.started":"2024-10-11T11:30:30.354816Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"positive_probs = meta_learner.predict_proba(stacked_val_preds.T)[:, 1]\nroc_auc_score(y_val, positive_probs)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:30.365895Z","iopub.status.busy":"2024-10-11T11:30:30.365581Z","iopub.status.idle":"2024-10-11T11:30:30.374986Z","shell.execute_reply":"2024-10-11T11:30:30.374021Z","shell.execute_reply.started":"2024-10-11T11:30:30.365861Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Final results\n","metadata":{}},{"cell_type":"markdown","source":"Therefore, the best results obtained are from the CNN and the VGG, the last one being slightly superior. However, as it takes considerably longer to train than the CNN, we will opt for this one, as the difference in performance is not so significant.\n\n\n\nWe will now train with the entire data set.\n","metadata":{}},{"cell_type":"code","source":"model = CNN(\n    in_chl=3,\n    out_size=2,\n    cv_args=[\n        (96, 5, 1, \"same\"),\n        (112, 3, 1, \"same\"),\n        (128, 3, 1, \"same\"),\n    ],\n    bt_norm=[True, True, False],\n    pl_args=[\n        (nn.MaxPool2d, 2, 2, 0),\n        (nn.MaxPool2d, 2, 1, 0),\n        (GlobalMaxPooling,),\n    ],\n    fc_args=[(96, 0.45), (80, 0.35), (64, 0.3)],\n).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nscheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.7**epoch)\n\ncnn_full = NeuralNetworkTrainer(\n    model,\n    criterion,\n    optimizer,\n    scheduler=scheduler,\n    device=device,\n).execute(\n    train_dataloader,\n    vdataloader=val_dataloader,\n    metric=roc_auc_score,\n    epochs=epochs,\n    early_stopper=EarlyStopper(**early_stopper_args),\n    verbose=2,\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-11T11:30:30.473066Z","iopub.status.busy":"2024-10-11T11:30:30.472739Z","iopub.status.idle":"2024-10-11T12:06:35.855007Z","shell.execute_reply":"2024-10-11T12:06:35.854034Z","shell.execute_reply.started":"2024-10-11T11:30:30.473033Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Since this is already the final model, we can detail the results a little more, for example, by showing the ROC curve, which is the metric we are using.\n","metadata":{}},{"cell_type":"code","source":"t_preds, t_y = cnn_full.predict(train_dataloader, has_target=True, calc_probs=True)\nv_preds, v_y = cnn_full.predict(val_dataloader, has_target=True, calc_probs=True)\n\npositive_train_preds = t_preds[:, 1]\npositive_val_preds = v_preds[:, 1]\n\n_, ax = plt.subplots(nrows=1, ncols=1)\nRocCurveDisplay.from_predictions(t_y, positive_train_preds, ax=ax, name=\"Train\")\nRocCurveDisplay.from_predictions(v_y, positive_val_preds, ax=ax, name=\"Validation\")\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We can also study which pixels have been the most relevant when making the prediction for each image.\n","metadata":{}},{"cell_type":"code","source":"idx = 7\n\nimg, label = orig_dataset[idx]\npreprocess = transforms.Compose(\n    [\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]\n)\nimg_tensor = preprocess(img).unsqueeze(0).to(device)\n\nmodel.eval()\nattributions = IntegratedGradients(model).attribute(\n    img_tensor,\n    baselines=torch.ones(img_tensor.shape, device=device),\n    target=1,\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_attributions(img, attributions):\n    attributions = attributions.squeeze().cpu().detach().numpy()\n    attributions = np.transpose(attributions, (1, 2, 0))\n\n    _, ax = plt.subplots(nrows=1, ncols=2)\n    ax[0].imshow(img)\n    ax[0].set_title(\"Original Image\")\n    ax[0].set_xticks([])\n    ax[0].set_yticks([])\n    ax[1].imshow(np.mean(np.abs(attributions), axis=2), cmap=\"inferno\")\n    ax[1].set_title(\"Integrated Gradients\")\n    ax[1].set_xticks([])\n    ax[1].set_yticks([])\n    plt.tight_layout()\n    plt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = np.argmax(model(img_tensor)[0].cpu().detach().numpy())\n\nprint(f\"True: {label} / Predicted: {pred}\")\nvisualize_attributions(img, attributions)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Finally, we create the file with the predictions on the test set.\n","metadata":{}},{"cell_type":"code","source":"img_dir = \"../input/histopathologic-cancer-detection/test\"\n\ntest_dataset = TifImageDataset(img_dir, transforms=preprocess)\n\ntest_dataloader = DataLoader(\n    test_dataset,\n    batch_size,\n    shuffle=False,\n    num_workers=mp.cpu_count(),\n    pin_memory=torch.cuda.is_available(),\n)\n\nprobs = cnn_full.predict(test_dataloader, calc_probs=True)[0]\n\npd.DataFrame(\n    list(zip(test_dataset.ids, probs[:, 1])),\n    columns=[\"id\", \"label\"],\n).to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}