{"metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"name": "python", "version": "3.12.13", "mimetype": "text/x-python", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py"}, "kaggle": {"accelerator": "none", "dataSources": [], "dockerImageVersionId": 28755, "isInternetEnabled": false, "language": "python", "sourceType": "notebook", "isGpuEnabled": false}}, "nbformat_minor": 4, "nbformat": 4, "cells": [{"metadata": {}, "cell_type": "markdown", "source": "# Train notebook\n\nTrain notebook for attending [Histopathologic Cancer Detection](https://www.kaggle.com/competitions/histopathologic-cancer-detection/overview) competition.\n\nRelated links:\n\nNotebook used for result submission, can be found \u2014 [here](https://www.kaggle.com/code/pavlozelinskiy/hcd-convnext-tiny-submit-v1).\n\nBest model dataset, can be found \u2014 [here](https://www.kaggle.com/datasets/pavlozelinskiy/hcd-convnext-tiny-weights-v1/data).\n\nCustom training dataset used in training, can be found \u2014 [here](https://www.kaggle.com/datasets/pavlozelinskiy/hcd-convnext-tiny-train-dataset-with-wsi-column).\n"}, {"cell_type": "code", "source": "%matplotlib inline\nimport numpy as np\nimport torch\nimport timm\nfrom torchvision import transforms as T\nimport torch.nn as nn\nfrom timm.data import resolve_model_data_config\nfrom sklearn.metrics import roc_auc_score\nimport os\nimport math\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_curve\nfrom torch.utils.data import DataLoader\n# from histopathologic_cancer_detection.dataset import ImageDataset\n# import hashlib # Note: used to check if there is duplicates in image\n\n# Debug\nimport inspect  # Note: i.e inspect.signature() / inspect.getdoc()\n# import kagglehub # Note: for more see https://github.com/Kaggle/kagglehub/blob/main/README.md\nfrom pprint import pprint  # Note: pretty print of json.files\n\n# Constants\nNUM_CLASSES = 1  # Note: bc of binary classification task\nDEVICE = 'cuda' # 12Gb VRam\nEPOCHS = 6\n\n# Config\nRUN_MODEL_TRAINING = True\nLOAD_SAVED_MODEL = False\n\nIS_KAGGLE = bool(os.environ.get('KAGGLE_KERNEL_RUN_TYPE'))\nKAGGLE_DIR = Path('/kaggle') if IS_KAGGLE else Path.cwd().parent / 'kaggle'\n\nINPUT_DIR = KAGGLE_DIR / 'input' / 'competitions' / 'histopathologic-cancer-detection'\nOUTPUT_DIR = KAGGLE_DIR / 'working'\nMODEL_DIR = OUTPUT_DIR\nNEW_DATASET_DIR  = (KAGGLE_DIR / 'input' / 'hcd-convnext-tiny-train-dataset-with-wsi-column') if IS_KAGGLE else OUTPUT_DIR\n\nBEST_MODEL_PATH = MODEL_DIR / 'best_model.pth'\nLAST_MODEL_PATH = MODEL_DIR / 'last_model.pth'\n\nfor label, d in [('INPUT_DIR', INPUT_DIR), ('MODEL_DIR', MODEL_DIR)]:\n    if not d.is_dir():\n        raise FileNotFoundError(\n            f\"{label} missing: {d}\\nmounted: {[p.name for p in Path('/kaggle/input').iterdir()]}\"\n            if IS_KAGGLE else f\"{label} missing: {d}\"\n        )", "metadata": {"ExecuteTime": {"end_time": "2026-09-15T18:54:18.494722Z", "start_time": "2026-09-15T18:54:15.071079200Z"}}, "outputs": [], "execution_count": 2}, {"metadata": {}, "cell_type": "markdown", "source": "# Define class for PyTorch Dataset\nto run training locally directly on Windows .py version of this class is used"}, {"metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": "# Copy pasted from dataset.py\nfrom torch.utils.data import Dataset\nfrom PIL import Image\n\nclass ImageDataset(Dataset):\n    def __init__(self, df, img_dir, tf, has_label=True):\n        self.ids = df['id'].values\n        self.y = df['label'].values.astype('float32') if has_label else None\n        self.dir = img_dir\n        self.tf = tf\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, i):\n        img = Image.open(f'{self.dir}/{self.ids[i]}.tif').convert('RGB')\n        img = self.tf(img)\n        return (img, self.y[i] if self.y is not None else img)  # TODO: Fix issue with (img, img)"}, {"cell_type": "markdown", "source": "# Dataset visualization", "metadata": {}}, {"cell_type": "code", "source": "# Note: Image resolution is 96 x 96\ndef visualize_data():\n    labels = pd.read_csv(f'{INPUT_DIR}/train_labels.csv')\n\n    with_tumor_condition = labels['label'] == 1\n    tumor_ids = labels[with_tumor_condition]['id'].head(5).tolist()\n    no_tumor_ids = labels[~with_tumor_condition]['id'].head(5).tolist()\n\n    fig, axes = plt.subplots(nrows=5, ncols=2, figsize=(8, 16))\n\n    def plot_image(axes, image_id, title_prefix):\n        filename = f\"{image_id}.tif\" if not str(image_id).endswith(\".tif\") else str(image_id)\n        img_path = os.path.join(f'{INPUT_DIR}/train/', filename)\n\n        try:\n            img = Image.open(img_path)\n            axes.imshow(img)\n            axes.set_title(f\"{title_prefix}\\nID: {str(image_id)[:8]}...\", fontsize=9)\n        except FileNotFoundError:\n            axes.text(0.5, 0.5, f\"Missing:\\n{filename}\", ha=\"center\", va=\"center\")\n        axes.axis(\"off\")\n\n    for ax, img_id in zip(axes[:, 0], tumor_ids):\n        plot_image(ax, img_id, \"Tumor (1)\")\n\n    for ax, img_id in zip(axes[:, 1], no_tumor_ids):\n        plot_image(ax, img_id, \"No Tumor (0)\")\n\n    plt.tight_layout()\n    plt.show()\n\n\nvisualize_data()", "metadata": {"trusted": true, "execution": {"iopub.status.busy": "2026-09-08T18:56:54.662200Z", "iopub.execute_input": "2026-09-08T18:56:54.662410Z", "iopub.status.idle": "2026-09-08T18:56:55.671885Z", "shell.execute_reply.started": "2026-09-08T18:56:54.662391Z", "shell.execute_reply": "2026-09-08T18:56:55.670804Z"}, "ExecuteTime": {"end_time": "2026-09-09T10:11:51.308714700Z", "start_time": "2026-09-09T10:11:50.800907600Z"}}, "outputs": [], "execution_count": 3}, {"metadata": {}, "cell_type": "markdown", "source": "# Dataset split\n(optional) check for duplicates using hashes"}, {"metadata": {"ExecuteTime": {"end_time": "2026-09-15T18:54:22.383571600Z", "start_time": "2026-09-15T18:54:22.011872Z"}}, "cell_type": "code", "source": "data_frame_corrected = pd.read_csv(f'{NEW_DATASET_DIR}/train_labels_wsi.csv')\nstratified_group_k_fold = StratifiedGroupKFold(n_splits=10, shuffle=True, random_state=42)\n# Note: Basically we pull only first train test split, ignoring the rest 9. This explains why `next` is used. Bc it returns iterator (generator), and we want only first pair.\n#  we do this, bc there is no direct alternative to split dataset with stratification and group which gives single pair.\ntrain_idx_array, validation_idx_array = next(stratified_group_k_fold.split(data_frame_corrected, data_frame_corrected['label'], groups=data_frame_corrected['wsi']))\n\ntrain_data_frame = data_frame_corrected.iloc[train_idx_array]\nvalidation_data_frame = data_frame_corrected.iloc[validation_idx_array]\n\n# sanity check\nassert set(train_data_frame['wsi']).isdisjoint(validation_data_frame['wsi'])\nprint(len(train_data_frame), len(validation_data_frame), validation_data_frame['wsi'].nunique())\n\n# Note: Quick check if dataset contains duplicated images\n# data_frame['hash'] = data_frame['id'].head(1000).map(lambda i: hashlib.md5(open(f'/kaggle/input/competitions/histopathologic-cancer-detection/train/{i}.tif', 'rb').read()).hexdigest())\n# print(data_frame['hash'].head(1000).duplicated().sum())", "outputs": [], "execution_count": 3}, {"metadata": {}, "cell_type": "markdown", "source": "# Training loop"}, {"metadata": {"ExecuteTime": {"end_time": "2026-09-15T17:17:46.175471700Z", "start_time": "2026-09-15T16:51:20.679735900Z"}}, "cell_type": "code", "source": "LR = 1e-4\nWEIGHT_DECAY = 1e-3\nMODEL_NAME = \"convnext_tiny.in12k_ft_in1k\"\nIMAGE_SIZE = 192\nACCUMULATION_STEPS = 2 # 128 * 2 = 256\n\nDROP_PATH_RATE = 0.0 # Note: 0.1 did not help\n\nmodel = timm.create_model(\n    model_name=MODEL_NAME,\n    pretrained=True,\n    num_classes=NUM_CLASSES,\n    drop_path_rate=DROP_PATH_RATE\n).to(DEVICE)\n\nconfig = timm.data.resolve_model_data_config(model)\ncheckpoint_config = {\n    'model_name': MODEL_NAME,\n    'num_classes': NUM_CLASSES,\n    'image_size': IMAGE_SIZE,\n    'mean': config['mean'],\n    'std': config['std'],\n}\n\ntransform_train = T.Compose([\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    T.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.05),\n    T.Resize(IMAGE_SIZE, interpolation=T.InterpolationMode.BICUBIC),\n    T.ToTensor(),\n    T.Normalize(config['mean'], config['std'])\n])\ntransform_validation = T.Compose([\n    T.Resize(IMAGE_SIZE, interpolation=T.InterpolationMode.BICUBIC),\n    T.ToTensor(),\n    T.Normalize(config['mean'], config['std'])\n])\n\ntrain_dataset = ImageDataset(train_data_frame, f'{INPUT_DIR}/train', transform_train)\nvalidation_dataset = ImageDataset(validation_data_frame, f'{INPUT_DIR}/train', transform_validation)\n\ntrain_loader = DataLoader(train_dataset, batch_size=256 // ACCUMULATION_STEPS, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True,\n                          persistent_workers=True)\n\nvalidation_loader = DataLoader(validation_dataset, batch_size=256, shuffle=False,\n                               num_workers=4, pin_memory=True,\n                               persistent_workers=True)\n\nloss_fn = nn.BCEWithLogitsLoss()  # Note: Other widely used name is criterion\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n# Note: torch.amp stands for Automatic Mixed Precision\nscaler = torch.amp.GradScaler()  # Note: Used for mixed precision training ()\n\nmetrics = {'train_loss': [], 'validation_loss': [], 'train_accuracy': [], 'validation_accuracy': [], 'validation_auc': []}\n\nstart_epoch = 0\nbest_auc = -1.0\n\nsteps_per_epoch = len(train_loader) // ACCUMULATION_STEPS # Note: calculate number of Optimizer steps\ntotal_steps = EPOCHS * steps_per_epoch\nwarmup_steps = int(0.05 * total_steps)\nassert total_steps != warmup_steps, \"Total steps cannot be equal to warmup steps\"\n\ndef lr_factor(step): # Note: Return coefficient which will be multiplied by LR\n    if step < warmup_steps:\n        return (step + 1) / warmup_steps # Note: Linear increase from 0->1\n    else:\n        progress = (step - warmup_steps) / (total_steps - warmup_steps)\n        return 0.5 * (1 + math.cos(math.pi * progress)) # Note: cosine decay 1->0\n\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_factor)\n\nif LOAD_SAVED_MODEL and os.path.exists(LAST_MODEL_PATH):\n    checkpoint = torch.load(LAST_MODEL_PATH, map_location=DEVICE, weights_only=True)\n    model.load_state_dict(checkpoint['model'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n    scaler.load_state_dict(checkpoint['scaler'])\n    start_epoch = checkpoint['epoch'] + 1\n    best_auc = checkpoint['best_auc']\n    metrics = checkpoint['metrics']\n    scheduler.load_state_dict(checkpoint['scheduler'])\n    print(f'resumed from epoch {checkpoint['epoch']}, best auc {best_auc:.4f}')\n\nif RUN_MODEL_TRAINING:\n    for epoch in range(start_epoch, EPOCHS):\n        model.train()\n        total_loss = torch.zeros((), device=DEVICE)\n        correct = torch.zeros((), device=DEVICE)\n        n_samples = 0\n\n        optimizer.zero_grad(set_to_none=True) # Note: once per epoch, and not inside batch loop, bc of gradient accumulation\n        for step , (x, y) in enumerate(train_loader):\n            x, y = x.to(DEVICE, non_blocking=True), y.to(DEVICE, non_blocking=True).float()  # Note: `non_blocking=True` won't work without `pin_memory=True`\n            # Note2: .float() is needed bc BCEWithLogitsLoss requires all values as float\n            with torch.amp.autocast('cuda'):  # Note: Used to force some operations to perform in float16 space\n                logits = model(x).squeeze(1)  # Note: squeeze bc BCEWithLogitsLoss requires to have x and y equal shape\n                loss = loss_fn(logits, y)  # Note2: specify `1` parameter explicitly, to avoid potential bugs\n            # Note: This is loss scaling\n            scaler.scale(loss / ACCUMULATION_STEPS).backward() # Note: Divide, bc of gradient accumulation\n\n            # Gradient accumulation\n            if (step + 1) % ACCUMULATION_STEPS == 0:\n                current_scale = scaler.get_scale()\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad(set_to_none=True)\n                should_skip_lr_scheduler = (current_scale > scaler.get_scale())\n                if not should_skip_lr_scheduler:\n                    scheduler.step()\n\n            batch_size = x.size(0)\n            n_samples += batch_size\n            total_loss += loss.item() * x.size(0)  # Note: Because last batch can be not full, we need to have real loss\n            correct += ((logits.detach() > 0) == y.bool()).sum()  # Note: this formula is equivalent to \"predicted\" in validation\n\n        train_loss = total_loss / n_samples\n        metrics['train_loss'].append(train_loss.item())  # Note: item used to transfer data from GPU to CPU\n        metrics['train_accuracy'].append((correct / n_samples).item())\n\n        model.eval()  # Note: This switch model to evaluation mode (but this won't disable pytorch autograd!)\n        total_loss = torch.zeros((), device=DEVICE)\n        n_samples = 0\n        predicted, Y = [], []\n        with torch.inference_mode():  # Note: This disables pytorch autograd\n            # Note2: With inference_mode() we can increase batch_size from 256 to 512\n            for x, y in validation_loader:\n                x, y = x.to(DEVICE), y.to(DEVICE).float()\n                with torch.amp.autocast('cuda'):\n                    logits = model(x).squeeze(1)\n                    loss = loss_fn(logits, y)\n                batch_size = x.size(0)\n                total_loss += loss.item() * batch_size\n                n_samples += batch_size\n                predicted.append(torch.sigmoid(\n                    logits.float()))  # Note: Sigmoid here is an analog of Softmax but for binary classification task\n                Y.append(y)\n\n        predicted = torch.cat(\n            predicted).cpu().numpy()  # Note: `.cpu()` bc scikit-learn can only operate on CPU AND to free up VRAM space\n        Y = torch.cat(Y).cpu().numpy()\n\n        # Calculate metrics\n        validation_loss = total_loss / n_samples\n        validation_accuracy = ((predicted > 0.5) == Y).mean()\n        validation_auc = roc_auc_score(Y, predicted)\n\n        metrics['validation_loss'].append(validation_loss.item())\n        metrics['validation_accuracy'].append(float(validation_accuracy))\n        metrics['validation_auc'].append(validation_auc)\n\n        print(f'ep {epoch} lr {optimizer.param_groups[0]['lr']} train {train_loss:.4f} val {validation_loss:.4f} '\n              f'acc {validation_accuracy:.4f} auc {validation_auc:.4f}')\n\n        # Save model\n        if validation_auc > best_auc:\n            best_auc = validation_auc\n            torch.save({\n                'model': model.state_dict(),\n                'epoch': epoch,\n                'validation_auc': validation_auc,\n                'config': checkpoint_config\n            }, BEST_MODEL_PATH)\n            print(f' ^^^ new best {validation_auc:.4f}')\n\n        torch.save({\n            'model': model.state_dict(),\n            'optimizer': optimizer.state_dict(),\n            'scaler': scaler.state_dict(),\n            'scheduler': scheduler.state_dict(),\n            'epoch': epoch,\n            'best_auc': best_auc,\n            'metrics': metrics,\n            'config': checkpoint_config\n        }, LAST_MODEL_PATH)", "outputs": [], "execution_count": 11}, {"metadata": {}, "cell_type": "markdown", "source": "# Plot metrics"}, {"cell_type": "code", "source": "fig, axes = plt.subplots(1, 3, figsize=(16, 4.5))\nepochs = range(1, len(metrics['train_loss']) + 1)\n\naxes[0].plot(epochs, metrics['train_loss'], 'o-', label='train')\naxes[0].plot(epochs, metrics['validation_loss'], 'o-', label='validation')\naxes[0].set_xlabel('epoch')\naxes[0].set_ylabel('BCEWithLogits loss')\naxes[0].set_title('Loss')\naxes[0].legend()\naxes[0].grid(alpha=.3)\n\naxes[1].plot(epochs, metrics['train_accuracy'], 'o-', label='train accuracy')\naxes[1].plot(epochs, metrics['validation_accuracy'], 'o-', label='validation  accuracy')\naxes[1].plot(epochs, metrics['validation_auc'], 'o-', label='validation  AUC')\naxes[1].set_xlabel('epoch')\naxes[1].set_title('Accuracy / AUC')\naxes[1].legend()\naxes[1].grid(alpha=.3)\n\nfpr, tpr, _ = roc_curve(Y, predicted)\naxes[2].plot(fpr, tpr, lw=2, label=f'AUC = {best_auc:.4f}')\naxes[2].plot([0, 1], [0, 1], 'k--', lw=1, label='random')\naxes[2].set_xlabel('False Positive Rate')\naxes[2].set_ylabel('True Positive Rate')\naxes[2].set_title('ROC (best model)')\naxes[2].legend()\naxes[2].grid(alpha=.3)\n\nplt.tight_layout()\nplt.show()", "metadata": {"trusted": true, "execution": {"iopub.status.busy": "2026-09-08T21:10:27.921492Z", "iopub.execute_input": "2026-09-08T21:10:27.921792Z", "iopub.status.idle": "2026-09-08T21:10:28.339852Z", "shell.execute_reply.started": "2026-09-08T21:10:27.921763Z", "shell.execute_reply": "2026-09-08T21:10:28.339243Z"}, "ExecuteTime": {"end_time": "2026-09-15T18:40:22.685710200Z", "start_time": "2026-09-15T18:40:22.390864200Z"}}, "outputs": [], "execution_count": 18}, {"metadata": {}, "cell_type": "markdown", "source": "# PCam metadata plus dataset join\nUsed to create dataset with 'wsi' column which reflects from which slide did specific sample was taken."}, {"metadata": {"ExecuteTime": {"end_time": "2026-09-14T15:54:25.475670500Z", "start_time": "2026-09-14T15:54:12.643133900Z"}}, "cell_type": "code", "source": "import h5py\nimport hashlib\n\ndef connect_with_wsi():\n    sha_to_meta = {}\n    for split in ['train', 'valid', 'test']:\n        x = h5py.File(f'{INPUT_DIR}/additional/camelyonpatch_level_2_split_{split}_x.h5', 'r')['x']\n        meta = pd.read_csv(f'{INPUT_DIR}/additional/camelyonpatch_level_2_split_{split}_meta.csv')\n        for i in range(x.shape[0]):\n            sha = hashlib.sha1(np.ascontiguousarray(x[i]).tobytes()).hexdigest() # Note: bc .tobytes() wants contiguous view\n            sha_to_meta[sha] = (meta.iloc[i]['wsi'], split)  # wsi only. We SHOULD SKIP tumor_path column as its answers.\n\n    sha_to_meta_default = (None, None)\n    labels = pd.read_csv(f'{INPUT_DIR}/train_labels.csv')\n    labels['wsi'] = labels['id'].map(lambda sha: sha_to_meta.get(sha, sha_to_meta_default)[0])\n    labels['pcam'] = labels['id'].map(lambda sha: sha_to_meta.get(sha, sha_to_meta_default)[1])\n    print(labels['wsi'].isna().sum(), labels['wsi'].nunique()) # Note: Discovered that there is 217 unique patches\n    print(labels['pcam'].value_counts())\n\n    labels[['id', 'label', 'wsi']].to_csv(f'{OUTPUT_DIR}/train_labels_wsi.csv', index=False)\n\n# connect_with_wsi()", "outputs": [], "execution_count": 3}]}