{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20604,"databundleVersionId":1357052,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ======================================================\n# Training Loop with Early Stopping (final training)\n# ======================================================\ndef train_model(model, criterion, optimizer, train_loader, val_loader, epochs=15, device=None,\n                patience=3, delta=1e-4, monitor=\"loss\", save_path=\"best_resnet50_earlystop.pth\", verbose=True):\n    \"\"\"\n    monitor: 'loss' or 'acc' -> chooses whether to monitor validation loss (min) or validation accuracy (max)\n    save_path: path to save best model weights when improvement occurs (None to disable)\n    \"\"\"\n    if device is None:\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    mode = \"min\" if monitor == \"loss\" else \"max\"\n    early_stopper = EarlyStopping(patience=patience, delta=delta, mode=mode, save_path=save_path)\n\n    for epoch in range(epochs):\n        model.train()\n        train_loss, correct, total = 0.0, 0, 0\n\n        for imgs, labels in train_loader:\n            imgs, labels = imgs.to(device), labels.to(device)\n\n            optimizer.zero_grad()\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            train_loss += loss.item() * imgs.size(0)\n            _, preds = torch.max(outputs, 1)\n            correct += (preds == labels).sum().item()\n            total += labels.size(0)\n\n        train_loss_avg = train_loss / total if total > 0 else 0.0\n        train_acc = correct / total if total > 0 else 0.0\n\n        # Validation\n        model.eval()\n        val_loss, val_correct, val_total = 0.0, 0, 0\n        with torch.no_grad():\n            for imgs, labels in val_loader:\n                imgs, labels = imgs.to(device), labels.to(device)\n                outputs = model(imgs)\n                loss = criterion(outputs, labels)\n\n                val_loss += loss.item() * imgs.size(0)\n                _, preds = torch.max(outputs, 1)\n                val_correct += (preds == labels).sum().item()\n                val_total += labels.size(0)\n\n        avg_val_loss = val_loss / val_total if val_total > 0 else 0.0\n        val_acc = val_correct / val_total if val_total > 0 else 0.0\n\n        if verbose:\n            print(f\"Epoch [{epoch+1}/{epochs}] Summary | \"\n                  f\"Train Loss: {train_loss_avg:.4f}, Train Acc: {train_acc:.4f}, \"\n                  f\"Val Loss: {avg_val_loss:.4f}, Val Acc: {val_acc:.4f}\")\n\n        # choose metric value to monitor\n        metric = avg_val_loss if monitor == \"loss\" else val_acc\n\n        # pass model so best weights are saved when improvement happens\n        if early_stopper(metric, model=model):\n            if save_path is not None:\n                print(f\"Early stopping triggered! Best model saved to: {save_path}\")\n            else:\n                print(\"Early stopping triggered!\")\n            break\n\n# ======================================================\n# Optuna Objective Function\n# ======================================================\ndef objective(trial):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    # Hyperparameters to tune\n    lr = trial.suggest_float(\"lr\", 1e-5, 1e-3, log=True)\n    batch_size = trial.suggest_categorical(\"batch_size\", [8, 16, 32])\n    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"SGD\"])\n\n    # Dataset + Loader\n    train_dataset = OSICDataset(train_df, data_dir, transform=transform)\n    val_dataset = OSICDataset(val_df, data_dir, transform=transform)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=2)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n\n    # Model\n    model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)\n    model.fc = nn.Linear(model.fc.in_features, 2)\n    model = model.to(device)\n\n    criterion = nn.CrossEntropyLoss()\n    if optimizer_name == \"Adam\":\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n    else:\n        optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)\n\n    accuracy = train_and_eval(model, criterion, optimizer, train_loader, val_loader, device, epochs=3)\n    return accuracy\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Run Optuna Study\n# ======================================================\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=10)\n\nbest_params = study.best_trial.params\nprint(\"Best hyperparameters:\", best_params)\nprint(\"Best accuracy:\", study.best_value)\n\n# ======================================================\n# Retrain Best Model (final)\n# ======================================================\nbest_params = study.best_trial.params\nprint(\"Best hyperparameters:\", best_params)\n\ntrain_dataset = OSICDataset(train_df, data_dir, transform=transform)\nval_dataset = OSICDataset(val_df, data_dir, transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=best_params[\"batch_size\"], shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=best_params[\"batch_size\"], shuffle=False, num_workers=2)\n\nmodel = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\nmodel.fc = nn.Linear(model.fc.in_features, 2)\nmodel = model.to(torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"))\n\ncriterion = nn.CrossEntropyLoss()\nif best_params[\"optimizer\"] == \"Adam\":\n    optimizer = torch.optim.Adam(model.parameters(), lr=best_params[\"lr\"])\nelse:\n    optimizer = torch.optim.SGD(model.parameters(), lr=best_params[\"lr\"], momentum=0.9)\n\n# --- final training: monitor='loss' (default). If you prefer to stop on val accuracy, set monitor='acc' ---\ntrain_model(model, criterion, optimizer, train_loader, val_loader, epochs=20,\n            patience=3, delta=1e-4, monitor=\"loss\", save_path=\"best_resnet50_earlystop.pth\", verbose=True)\n\n# If you want the final saved file to be named exactly as before, copy it:\ntry:\n    torch.save(model.state_dict(), \"best_resnet50.pth\")\n    print(\"Final model saved as best_resnet50.pth (latest weights).\")\nexcept Exception:\n    # saving model might have already been done by early stopper; ignore if not available\n    pass\n\nprint(\"Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-12T08:51:23.422624Z","iopub.execute_input":"2025-09-12T08:51:23.422862Z","iopub.status.idle":"2025-09-12T08:51:23.506709Z","shell.execute_reply.started":"2025-09-12T08:51:23.422842Z","shell.execute_reply":"2025-09-12T08:51:23.505227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Imports\n# ======================================================\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport numpy as np\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\n\n# ======================================================\n# 1️⃣ DICOM Loader (Fast)\n# ======================================================\ndef load_dicom_as_tensor(dcm_path, device):\n    dcm = pydicom.dcmread(dcm_path, force=True)\n    img = dcm.pixel_array.astype(np.float32)\n\n    # Normalize and resize\n    img = cv2.resize(img, (224, 224))\n    img = (img - img.min()) / (img.max() - img.min() + 1e-8)\n    img_rgb = np.repeat(img[..., None], 3, axis=-1)\n\n    # Transform to tensor\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ])\n    tensor = transform(img_rgb).unsqueeze(0).to(device)\n    return img_rgb, tensor\n\n# ======================================================\n# 2️⃣ Simple Grad-CAM Class (Optimized)\n# ======================================================\nclass FastGradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n\n        # Attach forward and backward hooks\n        target_layer.register_forward_hook(self.save_activation)\n        target_layer.register_full_backward_hook(self.save_gradient)\n\n    def save_activation(self, module, input, output):\n        self.activations = output.detach()\n\n    def save_gradient(self, module, grad_input, grad_output):\n        self.gradients = grad_output[0].detach()\n\n    def __call__(self, input_tensor, class_idx=None):\n        output = self.model(input_tensor)\n        if class_idx is None:\n            class_idx = torch.argmax(output, dim=1).item()\n\n        # Backward for target class\n        self.model.zero_grad()\n        score = output[0, class_idx]\n        score.backward()\n\n        # Compute Grad-CAM\n        weights = self.gradients.mean(dim=(2, 3), keepdim=True)\n        cam = torch.sum(weights * self.activations, dim=1).squeeze()\n        cam = F.relu(cam)\n        cam = cam - cam.min()\n        cam = cam / (cam.max() + 1e-8)\n        return cam.cpu().numpy()\n\n# ======================================================\n# 3️⃣ Load Model (Example: ResNet50)\n# ======================================================\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = models.resnet50(weights=None)\nmodel.fc = torch.nn.Linear(model.fc.in_features, 2)\nmodel = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\nmodel = model.to(device).eval()\n\n# ======================================================\n# 4️⃣ Select DICOM & Generate Heatmap\n# ======================================================\ndcm_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/12.dcm\"\n\norig_img, input_tensor = load_dicom_as_tensor(dcm_path, device)\n\ngradcam = FastGradCAM(model, model.layer4[-1])\nheatmap = gradcam(input_tensor)\n\n# ======================================================\n# 5️⃣ Overlay Heatmap (Fast)\n# ======================================================\ndef overlay_heatmap(img, heatmap, alpha=0.5):\n    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n    heatmap_color = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(heatmap_color, alpha, (img * 255).astype(np.uint8), 1 - alpha, 0)\n    plt.figure(figsize=(6,6))\n    plt.imshow(cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"🔥 Fast Grad-CAM\")\n    plt.show()\n\noverlay_heatmap(orig_img, heatmap)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T08:52:57.714414Z","iopub.execute_input":"2025-10-10T08:52:57.714688Z","iopub.status.idle":"2025-10-10T08:53:08.998010Z","shell.execute_reply.started":"2025-10-10T08:52:57.714630Z","shell.execute_reply":"2025-10-10T08:53:08.997199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##recent test \n# FULL GRAD-CAM + EXPLAINABILITY PIPELINE (AUTO SUMMARY)\nimport os\nimport time\nimport json\nimport numpy as np\nimport cv2\nimport pydicom\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport matplotlib.pyplot as plt\n\n\n# 1.DICOM Loader\n\ndef load_dicom_as_tensor(dcm_path, device):\n    \"\"\"Load DICOM file, normalize and convert to RGB tensor.\"\"\"\n    dcm = pydicom.dcmread(dcm_path, force=True)\n    img = dcm.pixel_array.astype(np.float32)\n\n    # Normalize and resize\n    img = cv2.resize(img, (224, 224))\n    img = (img - img.min()) / (img.max() - img.min() + 1e-8)\n    img_rgb = np.repeat(img[..., None], 3, axis=-1)\n\n    # Transform to tensor\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ])\n    tensor = transform(img_rgb).unsqueeze(0).to(device)\n    return img_rgb, tensor\n\n\n\n# 2.Fast Grad-CAM Class\n\nclass FastGradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n\n        # Attach hooks\n        target_layer.register_forward_hook(self.save_activation)\n        target_layer.register_full_backward_hook(self.save_gradient)\n\n    def save_activation(self, module, input, output):\n        self.activations = output.detach()\n\n    def save_gradient(self, module, grad_input, grad_output):\n        self.gradients = grad_output[0].detach()\n\n    def __call__(self, input_tensor, class_idx=None):\n        output = self.model(input_tensor)\n        if class_idx is None:\n            class_idx = torch.argmax(output, dim=1).item()\n\n        # Backward for target class\n        self.model.zero_grad()\n        score = output[0, class_idx]\n        score.backward()\n\n        # Compute Grad-CAM\n        weights = self.gradients.mean(dim=(2, 3), keepdim=True)\n        cam = torch.sum(weights * self.activations, dim=1).squeeze()\n        cam = F.relu(cam)\n        cam = cam - cam.min()\n        cam = cam / (cam.max() + 1e-8)\n        return cam.cpu().numpy()\n\n\n\n# 3.Load Model\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\nmodel = model.to(device).eval()\n\n\n#4.Load DICOM & Generate Heatmap\n\ndcm_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/12.dcm\"\norig_img, input_tensor = load_dicom_as_tensor(dcm_path, device)\n\ngradcam = FastGradCAM(model, model.layer4[-1])\nheatmap = gradcam(input_tensor)\n\n\n \n# 5.Overlay Heatmap\n \ndef overlay_heatmap(img, heatmap, alpha=0.5):\n    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n    heatmap_color = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(heatmap_color, alpha, (img * 255).astype(np.uint8), 1 - alpha, 0)\n    plt.figure(figsize=(6, 6))\n    plt.imshow(cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"🔥 Fast Grad-CAM\")\n    plt.show()\n    return overlay\n\noverlay_img = overlay_heatmap(orig_img, heatmap)\n\n\n\n# 6.Extract Top Hot Regions\n\ndef top_regions_from_heatmap(heatmap, topk=3, min_area_frac=0.01):\n    \"\"\"Extract top-K high-activation regions from a Grad-CAM heatmap.\"\"\"\n    H, W = heatmap.shape\n    regions = []\n    thresholds = [0.85, 0.6, 0.4]\n    idc = 0\n\n    for t in thresholds:\n        _, bw = cv2.threshold((heatmap * 255).astype('uint8'), int(t * 255), 255, cv2.THRESH_BINARY)\n        contours, _ = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n        for c in contours:\n            x, y, w, h = cv2.boundingRect(c)\n            area_frac = (w * h) / (W * H)\n            if area_frac < min_area_frac:\n                continue\n            x1, y1, x2, y2 = x / W, y / H, (x + w) / W, (y + h) / H\n            regions.append({\n                \"id\": f\"region_{idc}\",\n                \"bbox_normalized\": [round(x1, 3), round(y1, 3), round(x2, 3), round(y2, 3)],\n                \"avg_heat\": float(np.mean(heatmap[y:y+h, x:x+w])),\n                \"max_heat\": float(np.max(heatmap[y:y+h, x:x+w]))\n            })\n            idc += 1\n\n    regions = sorted(regions, key=lambda r: r[\"max_heat\"], reverse=True)\n    return regions[:topk]\n\n\n\n#7.Generate Automatic Heatmap Summary\n\ndef generate_heatmap_summary(heatmap):\n    \"\"\"Automatically generate a human-readable summary of Grad-CAM focus areas.\"\"\"\n    H, W = heatmap.shape\n    grid_y = np.array_split(np.arange(H), 3)\n    grid_x = np.array_split(np.arange(W), 3)\n\n    y_labels = [\"upper\", \"middle\", \"lower\"]\n    x_labels = [\"left\", \"central\", \"right\"]\n\n    avg_intensity = np.zeros((3, 3))\n    for i, ys in enumerate(grid_y):\n        for j, xs in enumerate(grid_x):\n            avg_intensity[i, j] = np.mean(heatmap[np.ix_(ys, xs)])\n\n    # Identify top 2 activated zones\n    flat_idx = np.argsort(avg_intensity.flatten())[::-1][:2]\n    summaries = []\n    for idx in flat_idx:\n        i, j = divmod(idx, 3)\n        summaries.append(f\"{y_labels[i]}-{x_labels[j]}\")\n\n    return \"Activation concentrated in \" + \" and \".join(summaries) + \" regions.\"\n\n\n\n# 8.Save Explainability JSON\n\nregions = top_regions_from_heatmap(heatmap)\nheatmap_summary = generate_heatmap_summary(heatmap)\n\nresult = {\n    \"image\": dcm_path,\n    \"explainability\": {\"method\": \"Grad-CAM\"},\n    \"model_info\": {\"name\": \"ResNet50\", \"weights\": \"ImageNet pretrained\"},\n    \"heatmap_summary\": heatmap_summary,\n    \"top_regions\": regions,\n    \"timestamp\": time.strftime(\"%Y-%m-%dT%H:%M:%SZ\", time.gmtime())\n}\n\nprint(\"Result\",result)\n\nwith open(\"gradcam_report.json\", \"w\") as f:\n    json.dump(result, f, indent=2)\n\nprint(\"✅ Grad-CAM report saved as gradcam_report.json\")\nprint(\"📋 Auto summary:\", heatmap_summary)\n\n\n\n#9.Draw Top Regions\n \ndef draw_regions(overlay_img, regions):\n    \"\"\"Visualize bounding boxes for top Grad-CAM regions.\"\"\"\n    H, W, _ = overlay_img.shape\n    img_draw = overlay_img.copy()\n    for reg in regions:\n        x1, y1, x2, y2 = reg[\"bbox_normalized\"]\n        x1, y1, x2, y2 = int(x1 * W), int(y1 * H), int(x2 * W), int(y2 * H)\n        cv2.rectangle(img_draw, (x1, y1), (x2, y2), (0, 255, 0), 2)\n        cv2.putText(img_draw, reg[\"id\"], (x1, y1 - 5),\n                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)\n    plt.figure(figsize=(6, 6))\n    plt.imshow(cv2.cvtColor(img_draw, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"📍 Top Activated Regions\")\n    plt.show()\n\ndraw_regions(overlay_img, regions)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T09:52:46.782278Z","iopub.execute_input":"2025-10-31T09:52:46.783118Z","iopub.status.idle":"2025-10-31T09:52:47.627768Z","shell.execute_reply.started":"2025-10-31T09:52:46.783089Z","shell.execute_reply":"2025-10-31T09:52:47.627099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def deletion_insertion_curve(model, img, heatmap, steps=50):\n    device = next(model.parameters()).device\n    img_tensor = torch.tensor(img.transpose(2,0,1)).unsqueeze(0).to(device).float()\n    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n    flat_idx = np.argsort(heatmap.flatten())[::-1]\n    \n    masks = np.ones_like(heatmap).flatten()\n    scores = []\n\n    with torch.no_grad():\n        for s in range(0, len(flat_idx), len(flat_idx)//steps):\n            masks[flat_idx[:s]] = 0\n            masked_img = img * masks.reshape(heatmap.shape)[..., None]\n            inp = transforms.ToTensor()(masked_img).unsqueeze(0).to(device)\n            score = torch.softmax(model(inp), dim=1).max().item()\n            scores.append(score)\n    return scores\n\n# Evaluate faithfulness\ndel_curve = deletion_insertion_curve(model, orig_img, heatmap)\nauc = np.trapz(del_curve) / len(del_curve)\n\n# Compute interpretability stats\nentropy = -np.sum(heatmap * np.log(heatmap + 1e-8))\nsparsity = np.mean(heatmap > 0.7)\n\nevaluation_metrics = {\n    \"AUC_deletion\": round(auc, 4),\n    \"Entropy\": round(entropy, 4),\n    \"Sparsity\": round(sparsity, 4)\n}\n\nprint(json.dumps(evaluation_metrics, indent=2, default=lambda o: float(o)))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T09:53:25.523698Z","iopub.execute_input":"2025-10-31T09:53:25.523993Z","iopub.status.idle":"2025-10-31T09:53:25.953052Z","shell.execute_reply.started":"2025-10-31T09:53:25.523973Z","shell.execute_reply":"2025-10-31T09:53:25.952351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Imports\n# ======================================================\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport numpy as np\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\n\n# ======================================================\n# 1️⃣ DICOM Loader (Fast)\n# ======================================================\ndef load_dicom_as_tensor(dcm_path, device):\n    dcm = pydicom.dcmread(dcm_path, force=True)\n    img = dcm.pixel_array.astype(np.float32)\n\n    # Normalize and resize\n    img = cv2.resize(img, (224, 224))\n    img = (img - img.min()) / (img.max() - img.min() + 1e-8)\n    img_rgb = np.repeat(img[..., None], 3, axis=-1)\n\n    # Transform to tensor\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ])\n    tensor = transform(img_rgb).unsqueeze(0).to(device)\n    return img_rgb, tensor\n\n# ======================================================\n# 2️⃣ Simple Grad-CAM Class (Optimized)\n# ======================================================\nclass FastGradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n\n        # Attach forward and backward hooks\n        target_layer.register_forward_hook(self.save_activation)\n        target_layer.register_full_backward_hook(self.save_gradient)\n\n    def save_activation(self, module, input, output):\n        self.activations = output.detach()\n\n    def save_gradient(self, module, grad_input, grad_output):\n        self.gradients = grad_output[0].detach()\n\n    def __call__(self, input_tensor, class_idx=None):\n        output = self.model(input_tensor)\n        if class_idx is None:\n            class_idx = torch.argmax(output, dim=1).item()\n\n        # Backward for target class\n        self.model.zero_grad()\n        score = output[0, class_idx]\n        score.backward()\n\n        # Compute Grad-CAM\n        weights = self.gradients.mean(dim=(2, 3), keepdim=True)\n        cam = torch.sum(weights * self.activations, dim=1).squeeze()\n        cam = F.relu(cam)\n        cam = cam - cam.min()\n        cam = cam / (cam.max() + 1e-8)\n        return cam.cpu().numpy()\n\n# ======================================================\n# 3️⃣ Load Model (Example: ResNet50)\n# ======================================================\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = models.resnet50(weights=None)\nmodel.fc = torch.nn.Linear(model.fc.in_features, 2)\nmodel = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\nmodel = model.to(device).eval()\n\n# ======================================================\n# 4️⃣ Select DICOM & Generate Heatmap\n# ======================================================\ndcm_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/12.dcm\"\n\norig_img, input_tensor = load_dicom_as_tensor(dcm_path, device)\n\ngradcam = FastGradCAM(model, model.layer4[-1])\nheatmap = gradcam(input_tensor)\n\n# ======================================================\n# 5️⃣ Overlay Heatmap (Fast)\n# ======================================================\ndef overlay_heatmap(img, heatmap, alpha=0.5):\n    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n    heatmap_color = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(heatmap_color, alpha, (img * 255).astype(np.uint8), 1 - alpha, 0)\n    plt.figure(figsize=(6,6))\n    plt.imshow(cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"🔥 Fast Grad-CAM\")\n    plt.show()\n\noverlay_heatmap(orig_img, heatmap)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T09:54:04.467452Z","iopub.execute_input":"2025-10-31T09:54:04.467707Z","iopub.status.idle":"2025-10-31T09:54:05.438138Z","shell.execute_reply.started":"2025-10-31T09:54:04.467688Z","shell.execute_reply":"2025-10-31T09:54:05.437415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# 🧠 FULL GRAD-CAM + EXPLAINABILITY PIPELINE (AUTO SUMMARY)\n# ======================================================\nimport os\nimport time\nimport json\nimport numpy as np\nimport cv2\nimport pydicom\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport matplotlib.pyplot as plt\n\n\n# ======================================================\n# 1️⃣ DICOM Loader\n# ======================================================\ndef load_dicom_as_tensor(dcm_path, device):\n    \"\"\"Load DICOM file, normalize and convert to RGB tensor.\"\"\"\n    dcm = pydicom.dcmread(dcm_path, force=True)\n    img = dcm.pixel_array.astype(np.float32)\n\n    # Normalize and resize\n    img = cv2.resize(img, (224, 224))\n    img = (img - img.min()) / (img.max() - img.min() + 1e-8)\n    img_rgb = np.repeat(img[..., None], 3, axis=-1)\n\n    # Transform to tensor\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.5]*3, [0.5]*3)\n    ])\n    tensor = transform(img_rgb).unsqueeze(0).to(device)\n    return img_rgb, tensor\n\n\n# ======================================================\n# 2️⃣ Fast Grad-CAM Class\n# ======================================================\nclass FastGradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n\n        # Attach hooks\n        target_layer.register_forward_hook(self.save_activation)\n        target_layer.register_full_backward_hook(self.save_gradient)\n\n    def save_activation(self, module, input, output):\n        self.activations = output.detach()\n\n    def save_gradient(self, module, grad_input, grad_output):\n        self.gradients = grad_output[0].detach()\n\n    def __call__(self, input_tensor, class_idx=None):\n        output = self.model(input_tensor)\n        if class_idx is None:\n            class_idx = torch.argmax(output, dim=1).item()\n\n        # Backward for target class\n        self.model.zero_grad()\n        score = output[0, class_idx]\n        score.backward()\n\n        # Compute Grad-CAM\n        weights = self.gradients.mean(dim=(2, 3), keepdim=True)\n        cam = torch.sum(weights * self.activations, dim=1).squeeze()\n        cam = F.relu(cam)\n        cam = cam - cam.min()\n        cam = cam / (cam.max() + 1e-8)\n        return cam.cpu().numpy()\n\n\n# ======================================================\n# 3️⃣ Load Model\n# ======================================================\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\nmodel = model.to(device).eval()\n\n\n# ======================================================\n# 4️⃣ Load DICOM & Generate Heatmap\n# ======================================================\ndcm_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264/13.dcm\"\norig_img, input_tensor = load_dicom_as_tensor(dcm_path, device)\n\ngradcam = FastGradCAM(model, model.layer4[-1])\nheatmap = gradcam(input_tensor)\n\n\n# ======================================================\n# 5️⃣ Overlay Heatmap\n# ======================================================\ndef overlay_heatmap(img, heatmap, alpha=0.5):\n    heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))\n    heatmap_color = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(heatmap_color, alpha, (img * 255).astype(np.uint8), 1 - alpha, 0)\n    plt.figure(figsize=(6, 6))\n    plt.imshow(cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"🔥 Fast Grad-CAM\")\n    plt.show()\n    return overlay\n\noverlay_img = overlay_heatmap(orig_img, heatmap)\n\n\n# ======================================================\n# 6️⃣ Extract Top Hot Regions\n# ======================================================\ndef top_regions_from_heatmap(heatmap, topk=3, min_area_frac=0.01):\n    \"\"\"Extract top-K high-activation regions from a Grad-CAM heatmap.\"\"\"\n    H, W = heatmap.shape\n    regions = []\n    thresholds = [0.85, 0.6, 0.4]\n    idc = 0\n\n    for t in thresholds:\n        _, bw = cv2.threshold((heatmap * 255).astype('uint8'), int(t * 255), 255, cv2.THRESH_BINARY)\n        contours, _ = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n        for c in contours:\n            x, y, w, h = cv2.boundingRect(c)\n            area_frac = (w * h) / (W * H)\n            if area_frac < min_area_frac:\n                continue\n            x1, y1, x2, y2 = x / W, y / H, (x + w) / W, (y + h) / H\n            regions.append({\n                \"id\": f\"region_{idc}\",\n                \"bbox_normalized\": [round(x1, 3), round(y1, 3), round(x2, 3), round(y2, 3)],\n                \"avg_heat\": float(np.mean(heatmap[y:y+h, x:x+w])),\n                \"max_heat\": float(np.max(heatmap[y:y+h, x:x+w]))\n            })\n            idc += 1\n\n    regions = sorted(regions, key=lambda r: r[\"max_heat\"], reverse=True)\n    return regions[:topk]\n\n\n# ======================================================\n# 7️⃣ Generate Automatic Heatmap Summary\n# ======================================================\ndef generate_heatmap_summary(heatmap):\n    \"\"\"Automatically generate a human-readable summary of Grad-CAM focus areas.\"\"\"\n    H, W = heatmap.shape\n    grid_y = np.array_split(np.arange(H), 3)\n    grid_x = np.array_split(np.arange(W), 3)\n\n    y_labels = [\"upper\", \"middle\", \"lower\"]\n    x_labels = [\"left\", \"central\", \"right\"]\n\n    avg_intensity = np.zeros((3, 3))\n    for i, ys in enumerate(grid_y):\n        for j, xs in enumerate(grid_x):\n            avg_intensity[i, j] = np.mean(heatmap[np.ix_(ys, xs)])\n\n    # Identify top 2 activated zones\n    flat_idx = np.argsort(avg_intensity.flatten())[::-1][:2]\n    summaries = []\n    for idx in flat_idx:\n        i, j = divmod(idx, 3)\n        summaries.append(f\"{y_labels[i]}-{x_labels[j]}\")\n\n    return \"Activation concentrated in \" + \" and \".join(summaries) + \" regions.\"\n\n\n# ======================================================\n# 8️⃣ Save Explainability JSON\n# ======================================================\nregions = top_regions_from_heatmap(heatmap)\nheatmap_summary = generate_heatmap_summary(heatmap)\n\nresult = {\n    \"image\": dcm_path,\n    \"explainability\": {\"method\": \"Grad-CAM\"},\n    \"model_info\": {\"name\": \"ResNet50\", \"weights\": \"ImageNet pretrained\"},\n    \"heatmap_summary\": heatmap_summary,\n    \"top_regions\": regions,\n    \"timestamp\": time.strftime(\"%Y-%m-%dT%H:%M:%SZ\", time.gmtime())\n}\n\nwith open(\"gradcam_report.json\", \"w\") as f:\n    json.dump(result, f, indent=2)\n\nprint(\"✅ Grad-CAM report saved as gradcam_report.json\")\nprint(\"📋 Auto summary:\", heatmap_summary)\n\n\n# ======================================================\n# 9️⃣ Optional: Draw Top Regions\n# ======================================================\ndef draw_regions(overlay_img, regions):\n    \"\"\"Visualize bounding boxes for top Grad-CAM regions.\"\"\"\n    H, W, _ = overlay_img.shape\n    img_draw = overlay_img.copy()\n    for reg in regions:\n        x1, y1, x2, y2 = reg[\"bbox_normalized\"]\n        x1, y1, x2, y2 = int(x1 * W), int(y1 * H), int(x2 * W), int(y2 * H)\n        cv2.rectangle(img_draw, (x1, y1), (x2, y2), (0, 255, 0), 2)\n        cv2.putText(img_draw, reg[\"id\"], (x1, y1 - 5),\n                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)\n    plt.figure(figsize=(6, 6))\n    plt.imshow(cv2.cvtColor(img_draw, cv2.COLOR_BGR2RGB))\n    plt.axis(\"off\")\n    plt.title(\"📍 Top Activated Regions\")\n    plt.show()\n\ndraw_regions(overlay_img, regions)\n\n# ======================================================\n# ⚠️ Disclaimer\n# ======================================================\nprint(\"⚠️ This visualization shows model attention — not a medical diagnosis. Always confirm with a radiologist.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T09:54:42.754950Z","iopub.execute_input":"2025-10-31T09:54:42.755514Z","iopub.status.idle":"2025-10-31T09:54:43.518225Z","shell.execute_reply.started":"2025-10-31T09:54:42.755491Z","shell.execute_reply":"2025-10-31T09:54:43.517497Z"}},"outputs":[],"execution_count":null}]}