{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"},{"sourceId":212102090,"sourceType":"kernelVersion"},{"sourceId":212160894,"sourceType":"kernelVersion"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##### Histopathologic Cancer Detection - Testing and Submission Generation\n##### Author: Aaron Storey\n##### Date: December 09, 2024\n##### Version: 1.0\n\nThis notebook implements the inference pipeline for the histopathologic cancer detection model\nand generates submission files for Kaggle. Uses best performing EfficientNet model (V13: 0.8682).","metadata":{}},{"cell_type":"markdown","source":"#### 1. Install and Import Required Packages","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet_pytorch albumentations --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T20:24:18.097651Z","iopub.execute_input":"2024-12-09T20:24:18.097899Z","iopub.status.idle":"2024-12-09T20:24:30.521249Z","shell.execute_reply.started":"2024-12-09T20:24:18.097872Z","shell.execute_reply":"2024-12-09T20:24:30.520261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom PIL import Image\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom efficientnet_pytorch import EfficientNet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T20:24:30.523047Z","iopub.execute_input":"2024-12-09T20:24:30.523359Z","iopub.status.idle":"2024-12-09T20:24:35.743523Z","shell.execute_reply.started":"2024-12-09T20:24:30.523329Z","shell.execute_reply":"2024-12-09T20:24:35.742627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 2. Configuration","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/histopathologic-cancer-detection\"\nTEST_DIR = f\"{DATA_DIR}/test\"\nMODEL_PATH = \"/kaggle/input/as-histopathologic-cancer-02-training/model_best.pth\"  # Adjust this if needed\nSUBMISSION_FILE = \"submission.csv\"\nTHRESHOLD = 0.5\n\nTARGET_SIZE = (96, 96)\nBATCH_SIZE = 64\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T20:24:35.744771Z","iopub.execute_input":"2024-12-09T20:24:35.745330Z","iopub.status.idle":"2024-12-09T20:24:35.841302Z","shell.execute_reply.started":"2024-12-09T20:24:35.745288Z","shell.execute_reply":"2024-12-09T20:24:35.840348Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 3. Model Definition and Loading","metadata":{}},{"cell_type":"code","source":"class CancerClassifier(nn.Module):\n    def __init__(self, num_classes=2):\n        super(CancerClassifier, self).__init__()\n        self.model = EfficientNet.from_pretrained(\"efficientnet-b0\")\n        self.model._fc = nn.Linear(self.model._fc.in_features, num_classes)\n\n    def forward(self, x):\n        return self.model(x)\n\nmodel = CancerClassifier().to(device)\nmodel.load_state_dict(torch.load(MODEL_PATH))\nmodel.eval()\nprint(\"Loaded best model.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 4. Test Dataset and Transforms","metadata":{}},{"cell_type":"code","source":"class HistologyTestDataset(Dataset):\n    def __init__(self, img_ids, img_dir, transform):\n        self.img_ids = img_ids\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.img_ids)\n\n    def __getitem__(self, idx):\n        img_id = self.img_ids[idx]\n        img_path = os.path.join(self.img_dir, f\"{img_id}.tif\")\n        img = np.array(Image.open(img_path).convert(\"RGB\"))\n        img = self.transform(image=img)['image']\n        return img, img_id\n\ntest_transforms = A.Compose([\n    A.Resize(*TARGET_SIZE),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2(),\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 5. Prepare Test DataLoader","metadata":{}},{"cell_type":"code","source":"test_img_ids = [p.stem for p in Path(TEST_DIR).glob(\"*.tif\")]\nprint(f\"Number of test images: {len(test_img_ids)}\")\n\ntest_dataset = HistologyTestDataset(test_img_ids, TEST_DIR, test_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### 6. Inference and Submission","metadata":{}},{"cell_type":"code","source":"model.eval()\npredictions = []\nids = []\n\nwith torch.no_grad():\n    for images, img_ids in tqdm(test_loader, desc=\"Generating predictions\"):\n        images = images.to(device)\n        outputs = model(images)\n        probs = torch.softmax(outputs, dim=1)[:, 1].cpu().numpy()\n        predictions.extend(probs)\n        ids.extend(img_ids)\n\nlabels = (np.array(predictions) > THRESHOLD).astype(int)\nsubmission_df = pd.DataFrame({\"id\": ids, \"label\": labels})\nsubmission_df.to_csv(SUBMISSION_FILE, index=False)\nprint(f\"Submission file '{SUBMISSION_FILE}' created.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}