{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport time\nfrom tqdm import tqdm\nfrom collections import OrderedDict\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport PIL\n\nfrom numpy.random import choice\n\n\nfrom sklearn.metrics import roc_auc_score\n\n\nimport torch\nfrom torch import nn, Tensor\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nimport torchvision\n\nnp.random.seed(14)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:02.534769Z","iopub.execute_input":"2024-12-29T06:29:02.53499Z","iopub.status.idle":"2024-12-29T06:29:07.391903Z","shell.execute_reply.started":"2024-12-29T06:29:02.534968Z","shell.execute_reply":"2024-12-29T06:29:07.390992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = \"/kaggle/input/histopathologic-cancer-detection\"\n\ntrain_dir = os.path.join(data_dir,\"train\" )\ntest_dir = os.path.join(data_dir, \"test\")\n\nprint(f\"training data : {len(os.listdir(train_dir))}, test data: {len(os.listdir(test_dir))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:07.392845Z","iopub.execute_input":"2024-12-29T06:29:07.393275Z","iopub.status.idle":"2024-12-29T06:29:30.517642Z","shell.execute_reply.started":"2024-12-29T06:29:07.393251Z","shell.execute_reply":"2024-12-29T06:29:30.516673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_val_split(df, val_ratio):\n    val_size = int(len(df)*val_ratio)\n    val_ids = np.random.choice(df[\"id\"], size = val_size, replace=False)\n\n    val_df = df[df['id'].isin(val_ids)]\n    train_df = df[~df['id'].isin(val_ids)]\n \n    return train_df, val_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:30.518476Z","iopub.execute_input":"2024-12-29T06:29:30.518802Z","iopub.status.idle":"2024-12-29T06:29:30.523457Z","shell.execute_reply.started":"2024-12-29T06:29:30.518774Z","shell.execute_reply":"2024-12-29T06:29:30.52246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(data_dir, \"train_labels.csv\"))\ntrain_label_df, val_label_df = train_val_split(df, val_ratio = 0.2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:30.524189Z","iopub.execute_input":"2024-12-29T06:29:30.524432Z","iopub.status.idle":"2024-12-29T06:29:30.964188Z","shell.execute_reply.started":"2024-12-29T06:29:30.524413Z","shell.execute_reply":"2024-12-29T06:29:30.963166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def function_timer(function):\n    def wrapper(*args, **kwargs):\n        start = time.time()\n        result = function(*args, **kwargs)\n        duration = time.time - start\n        hours = int(duration//60**2)\n        minutes = int((duration%60**2)//60)\n        seconds = int(duration%60)\n        print(f\"Execution time for function {function.__name__} : {hours}:{minutes}:{seconds}\")\n\n        return result\n    return wrapper","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:30.965237Z","iopub.execute_input":"2024-12-29T06:29:30.965567Z","iopub.status.idle":"2024-12-29T06:29:30.971678Z","shell.execute_reply.started":"2024-12-29T06:29:30.965538Z","shell.execute_reply":"2024-12-29T06:29:30.970792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class HistoPatches(Dataset):\n    def __init__(self,\n                image_dir:str,\n                label_df=None,\n                transform= transforms.ToTensor(),\n                sample_n= None,\n                in_memory=False):\n        self.image_dir = image_dir\n        self.label_df = label_df\n        self.transform = transform\n        self.in_memory = in_memory\n\n        if label_df is not None:\n            if sample_n:\n                self.label_df = self.label_df.sample(n=sample_n)\n            ids = set(self.label_df.id)\n            self.img_files = [f for f in os.listdir(image_dir) if f.split(\".\")[0] in ids]\n        else:\n            if sample_n is not None:\n                print('subsampling is currently only implemented when a label-dataframe is provided.')\n                return\n            self.img_files = os.listdir(image_dir)\n\n        if in_memory:\n            self.id2image = self._load_images()\n\n        print(f'Initialized datatset with {len(self.img_files)} images.\\n')\n\n    @function_timer\n\n    def _load_images(self):\n        print('loading images in memory...')\n        id2image = {}\n\n        for file_name in self.img_files:\n            img = PIL.Image.open(os.path.join(self.image_dir, file_name))\n            X = self.transform(img)\n            id_ = file_name.split(\".\")[0]\n            id2image[id_] = X\n        return id2image\n\n\n    def __getitem__(self, idx):\n        file_name = self.img_files[idx]\n        id_ = file_name.split(\".\")[0]\n\n        if self.in_memory:\n            X = self.id2image[id_]\n        else:\n            img = PIL.Image.open(os.path.join(self.image_dir, file_name))\n            X = self.transform(img)\n            \n        if self.label_df is not None:\n            y = float(self.label_df.loc[self.label_df['id']==id_, 'label'].iloc[0])\n            return X, y\n        else:\n            return X, id_\n\n    def __len__(self):\n        return len(self.img_files)\n\nmemory = False\nbatchsize = 64\nimage_trans = transforms.Compose([transforms.ToTensor(),\n                                 transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061], \n                                                       std=[0.22246036, 0.26757348, 0.19798167])\n                                ])\n\ntrain_trans = transforms.Compose([transforms.ToTensor(),\n                                  transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061], \n                                                       std=[0.22246036, 0.26757348, 0.19798167]),\n                                  transforms.RandomRotation((-180, 180)),\n                                  transforms.RandomHorizontalFlip()])\n            \n\ntrain = HistoPatches(train_dir,\n                    train_label_df,\n                    transform = train_trans,\n                    in_memory = memory)\nval = HistoPatches(train_dir,\n                    val_label_df,\n                    transform = image_trans,\n                    in_memory = memory)\n\ntrain_loader = DataLoader(train, batch_size = batchsize, shuffle=True)\nval_loader = DataLoader(val, batch_size = batchsize, shuffle = True)\n\nX, y = next(iter(train_loader))\nprint('batch-dimensions: ', X.shape, y.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:30.973777Z","iopub.execute_input":"2024-12-29T06:29:30.974006Z","iopub.status.idle":"2024-12-29T06:29:36.181405Z","shell.execute_reply.started":"2024-12-29T06:29:30.973987Z","shell.execute_reply":"2024-12-29T06:29:36.180479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n        super(ResidualBlock, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.conv3 = nn.Conv2d(out_channels, out_channels * 4, kernel_size=1, stride=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(out_channels * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n\n    def forward(self, x):\n        identity = x\n        if self.downsample:\n            identity = self.downsample(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n        out = self.bn3(out)\n        out += identity\n        out = self.relu(out)\n        return out\n\nclass ResNet50(nn.Module):\n    def __init__(self, num_classes=1):\n        super(ResNet50, self).__init__()\n        self.in_channels = 64\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        \n        # Define ResNet-50 architecture\n        self.layer1 = self._make_layer(64, 3)\n        self.layer2 = self._make_layer(128, 4, stride=2)\n        self.layer3 = self._make_layer(256, 6, stride=2)\n        self.layer4 = self._make_layer(512, 3, stride=2)\n        \n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * 4, num_classes)\n\n    def _make_layer(self, out_channels, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.in_channels != out_channels * 4:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.in_channels, out_channels * 4, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels * 4),\n            )\n        layers = [ResidualBlock(self.in_channels, out_channels, stride, downsample)]\n        self.in_channels = out_channels * 4\n        for _ in range(1, blocks):\n            layers.append(ResidualBlock(self.in_channels, out_channels))\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n        return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:36.182507Z","iopub.execute_input":"2024-12-29T06:29:36.182769Z","iopub.status.idle":"2024-12-29T06:29:36.195714Z","shell.execute_reply.started":"2024-12-29T06:29:36.182747Z","shell.execute_reply":"2024-12-29T06:29:36.194876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = ResNet50(num_classes=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:36.196537Z","iopub.execute_input":"2024-12-29T06:29:36.196851Z","iopub.status.idle":"2024-12-29T06:29:36.300569Z","shell.execute_reply.started":"2024-12-29T06:29:36.196821Z","shell.execute_reply":"2024-12-29T06:29:36.299893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@function_timer\ndef train_model(net, train, validation, optimizer, device, max_epoch=100, verbose=False):\n    global error_df\n    error_df = pd.DataFrame(columns=['train_bce', 'train_acc', 'train_auc', 'val_bce', 'val_acc', 'val_auc'])\n    criterion = nn.BCEWithLogitsLoss()\n    scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n    net.to(device)\n\n    print('epoch\\tLR\\ttr-BCE\\ttr-Acc\\ttr-AUC\\t\\tval-BCE\\tval-Acc\\tval-AUC')\n\n    for epoch in range(max_epoch):\n        net.train()\n        training_bce = training_acc= training_auc = 0\n\n        for X, y in train:\n            X, y = X.to(device), y.to(device)\n            optimizer.zero_grad()\n            out = net(X).squeeze()\n\n            labels = y.detach().cpu().numpy()\n            probabilities = torch.sigmoid(out).detach().cpu().numpy()\n            predictions = probabilities.round()\n            loss = criterion(out.type(torch.DoubleTensor).cuda(), y)\n\n            training_bce+=loss.item()\n            training_acc += np.mean(labels == predictions)*100\n            training_auc += roc_auc_score(y_true=labels, y_score=probabilities)\n            loss.backward()\n            optimizer.step()\n\n        with torch.no_grad():\n            net.eval()\n            validation_bce = validation_acc = validation_auc = 0\n\n            for X,y in validation:\n                X, y = X.to(device), y.to(device)\n                out = net(X).squeeze()\n                labels = y.detach().cpu().numpy()\n                probabilities = torch.sigmoid(out).detach().cpu().numpy()\n                predictions = probabilities.round()\n                loss = criterion(out.type(torch.DoubleTensor).cuda(), y)\n                validation_bce+=loss.item()\n                validation_acc += np.mean(labels == predictions)*100\n                validation_auc += roc_auc_score(y_true=labels, y_score=probabilities)\n    \n        # convert to batch loss:\n        training_bce   /= len(train)\n        training_acc   /= len(train)\n        training_auc   /= len(train)\n        \n        validation_bce /= len(validation)\n        validation_acc /= len(validation)\n        validation_auc /= len(validation)\n        scheduler.step()\n       \n        #torch.save(net.state_dict(), f'epoch{epoch}.pt')\n        error_stats = [training_bce, training_acc, training_auc, validation_bce, validation_acc, validation_auc]\n        new_row = pd.DataFrame([error_stats], columns=error_df.columns)\n        error_df = pd.concat([error_df, new_row], ignore_index=True)\n\n        print('{}\\t{:.4f}\\t{:.4f}\\t{:.2f}\\t{:.4f}\\t\\t{:.4f}\\t{:.2f}\\t{:.4f}'.format(epoch, optimizer.param_groups[0]['lr'], *error_stats))\n\n\n                \n                    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:29:36.301344Z","iopub.execute_input":"2024-12-29T06:29:36.301668Z","iopub.status.idle":"2024-12-29T06:29:36.310832Z","shell.execute_reply.started":"2024-12-29T06:29:36.301638Z","shell.execute_reply":"2024-12-29T06:29:36.309988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device=torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:30:35.061046Z","iopub.execute_input":"2024-12-29T06:30:35.061354Z","iopub.status.idle":"2024-12-29T06:30:35.067281Z","shell.execute_reply.started":"2024-12-29T06:30:35.061324Z","shell.execute_reply":"2024-12-29T06:30:35.066521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0) \n\ntrain_model(model,\n            train_loader,\n            val_loader,\n            optimizer,\n            device=torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'),\n            max_epoch=5,\n            verbose=False)\ntorch.save(model.state_dict(), 'swag_net.pt')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:31:05.368395Z","iopub.execute_input":"2024-12-29T06:31:05.368747Z","execution_failed":"2024-12-29T07:21:44.946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_and_display(net, testset, n=10):\n\n    net.eval()\n    for i in choice(range(len(testset)), size=n):\n        image, label = test_set[i]\n        output = net(image.unsqueeze(0).cuda())\n        prob, pred = prediction_from_output(output.unsqueeze(0))\n        prob, pred = prob.item(), pred.item()\n        evaluation = 'correct' if pred == label else 'mistake'\n\n        plt.figure( figsize=(2, 2) )\n        print(f'\\ntruth: {label} | pred: {pred} | prob: {prob:.2f}')\n        print(f'{evaluation}: ({class_name(label)} vs. {class_name(pred)})')\n        show_image(image, means=channel_means, stdevs=channel_standard_devs)\n\n\ndef plot_error_curves(training_error, validation_error, error_name='error', ylim=None, save_fig=True):\n\n    assert len(training_error) == len(validation_error) > 1\n    \n    fig, ax = plt.subplots()\n    ax.plot(range(len(training_error)), training_error)\n    ax.plot(range(len(validation_error)), validation_error)\n    \n    if ylim:\n        ax.set_ylim(*ylim)\n    \n    ax.set_xlabel('epoch')\n    ax.set_ylabel('CE')\n    ax.legend(('training', 'validation'))\n    ax.set_title(f'{error_name} over time')\n    \n    if save_fig:\n        fig.savefig(f'{error_name.replace(\" \", \"_\")}_learning_curve', bbox_inches='tight', transparent=True)\n    \n    plt.show();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:30:15.76181Z","iopub.status.idle":"2024-12-29T06:30:15.762052Z","shell.execute_reply":"2024-12-29T06:30:15.761955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" plot_error_curves(error_df.train_bce, error_df.val_bce, error_name='Binary Cross Entropy', ylim=(0, 1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:30:15.762933Z","iopub.status.idle":"2024-12-29T06:30:15.763267Z","shell.execute_reply":"2024-12-29T06:30:15.76314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_error_curves(error_df.train_acc, error_df.val_acc, error_name='Accuracy')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:30:15.763799Z","iopub.status.idle":"2024-12-29T06:30:15.764034Z","shell.execute_reply":"2024-12-29T06:30:15.763939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_error_curves(error_df.train_auc, error_df.val_auc, error_name='Area Under the Curve')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T06:30:15.764902Z","iopub.status.idle":"2024-12-29T06:30:15.765209Z","shell.execute_reply":"2024-12-29T06:30:15.765044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}