{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13577267,"sourceType":"datasetVersion","datasetId":8625201}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PhysioNet ECG U-Net Inference\n\nThis notebook uses a trained U-Net model for ECG waveform segmentation.\n\n**Model Details:**\n- Architecture: U-Net (31M parameters)\n- Training: 30 epochs on 8,793 images\n- Best Validation Loss: 0.0525\n- Model Size: 356MB","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pathlib import Path\nfrom PIL import Image\nimport cv2\nfrom tqdm import tqdm\n\nprint(f'PyTorch version: {torch.__version__}')\nprint(f'CUDA available: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T12:33:36.941896Z","iopub.execute_input":"2025-11-01T12:33:36.942154Z","iopub.status.idle":"2025-11-01T12:33:40.849590Z","shell.execute_reply.started":"2025-11-01T12:33:36.942121Z","shell.execute_reply":"2025-11-01T12:33:40.848946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load U-Net Model","metadata":{}},{"cell_type":"code","source":"class DoubleConv(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        return self.conv(x)\n\nclass Down(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.pool_conv = nn.Sequential(\n            nn.MaxPool2d(2),\n            DoubleConv(in_channels, out_channels)\n        )\n    \n    def forward(self, x):\n        return self.pool_conv(x)\n\nclass Up(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.up = nn.ConvTranspose2d(in_channels, in_channels // 2, 2, stride=2)\n        self.conv = DoubleConv(in_channels, out_channels)\n    \n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        diffY = x2.size()[2] - x1.size()[2]\n        diffX = x2.size()[3] - x1.size()[3]\n        x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,\n                       diffY // 2, diffY - diffY // 2])\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\nclass UNet(nn.Module):\n    def __init__(self, n_channels=3, n_classes=2):\n        super().__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        \n        # Encoder\n        self.inc = DoubleConv(n_channels, 64)\n        self.down1 = Down(64, 128)\n        self.down2 = Down(128, 256)\n        self.down3 = Down(256, 512)\n        self.down4 = Down(512, 1024)\n        \n        # Decoder (FIXED)\n        self.up1 = Up(1024, 512)\n        self.up2 = Up(512, 256)\n        self.up3 = Up(256, 128)\n        self.up4 = Up(128, 64)\n        \n        # Output\n        self.outc = nn.Conv2d(64, n_classes, 1)\n    \n    def forward(self, x):\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x5 = self.down4(x4)\n        \n        x = self.up1(x5, x4)\n        x = self.up2(x, x3)\n        x = self.up3(x, x2)\n        x = self.up4(x, x1)\n        \n        return self.outc(x)\n\n# Load model\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = UNet(n_channels=3, n_classes=2)\n\n# Load weights from dataset\nmodel_path = '/kaggle/input/ecg-unet-trained-model/unet_best.pth'\n\ncheckpoint = torch.load(model_path, map_location=device)\n\n  # Handle both checkpoint formats: dict with 'model_state_dict' or direct state_dict\nif isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:\n  model.load_state_dict(checkpoint['model_state_dict'])\n  print(f'Loaded checkpoint from epoch {checkpoint.get(\"epoch\", \"unknown\")}')\n  print(f'Training loss: {checkpoint.get(\"train_loss\", \"N/A\"):.4f}')\n  print(f'Validation loss: {checkpoint.get(\"val_loss\", \"N/A\"):.4f}')\nelse:\n  model.load_state_dict(checkpoint)\n  \nmodel = model.to(device)\nmodel.eval()\n\nprint(f'Model loaded successfully!')\nprint(f'Parameters: {sum(p.numel() for p in model.parameters()):,}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T12:47:38.954036Z","iopub.execute_input":"2025-11-01T12:47:38.954601Z","iopub.status.idle":"2025-11-01T12:47:39.664607Z","shell.execute_reply.started":"2025-11-01T12:47:38.954576Z","shell.execute_reply":"2025-11-01T12:47:39.663899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocessing Function","metadata":{}},{"cell_type":"code","source":"def preprocess_image(image_path, target_size=(512, 1024)):\n    \"\"\"Preprocess ECG image for U-Net inference.\"\"\"\n    # Read image\n    img = cv2.imread(str(image_path))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    # Resize\n    img = cv2.resize(img, (target_size[1], target_size[0]))\n    \n    # Normalize to [0, 1]\n    img = img.astype(np.float32) / 255.0\n    \n    # Convert to tensor (C, H, W)\n    img_tensor = torch.from_numpy(img.transpose(2, 0, 1))\n    \n    return img_tensor.unsqueeze(0)  # Add batch dimension\n\ndef predict_mask(model, image_tensor, device):\n    \"\"\"Run inference and get segmentation mask.\"\"\"\n    with torch.no_grad():\n        image_tensor = image_tensor.to(device)\n        output = model(image_tensor)\n        \n        # Apply sigmoid for probabilities\n        probs = torch.sigmoid(output)\n        \n        # Get binary mask (threshold at 0.5)\n        mask = (probs > 0.5).float()\n        \n    return mask.cpu().numpy()[0]  # Return (2, H, W)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T12:47:48.998537Z","iopub.execute_input":"2025-11-01T12:47:48.999253Z","iopub.status.idle":"2025-11-01T12:47:49.004749Z","shell.execute_reply.started":"2025-11-01T12:47:48.999230Z","shell.execute_reply":"2025-11-01T12:47:49.003986Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference on Test Data","metadata":{}},{"cell_type":"code","source":"# Load test data\ntest_dir = Path('/kaggle/input/physionet-ecg-image-digitization/test')\ntest_images = sorted(test_dir.glob('*.png'))\n\nprint(f'Found {len(test_images)} test images')\n\n# Run inference on a sample\nif len(test_images) > 0:\n    sample_img = test_images[0]\n    print(f'Testing on: {sample_img.name}')\n    \n    # Preprocess\n    img_tensor = preprocess_image(sample_img)\n    \n    # Predict\n    mask = predict_mask(model, img_tensor, device)\n    \n    print(f'Mask shape: {mask.shape}')\n    print(f'Waveform mask (channel 0): {mask[0].sum()} pixels')\n    print(f'Grid mask (channel 1): {mask[1].sum()} pixels')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T12:47:52.000841Z","iopub.execute_input":"2025-11-01T12:47:52.001123Z","iopub.status.idle":"2025-11-01T12:47:52.737329Z","shell.execute_reply.started":"2025-11-01T12:47:52.001103Z","shell.execute_reply":"2025-11-01T12:47:52.736515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Generate Submission\n\n**Note:** This is a basic inference example. For full submission, you need to:\n1. Extract signal from segmentation mask\n2. Apply calibration and scaling\n3. Convert to competition format (CSV with id, value columns)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom scipy.interpolate import interp1d\n\n# ============================================================================\n# CALIBRATION DETECTION (handles small/large grid correctly)\n# ============================================================================\n\ndef detect_horizontal_lines(grid_mask):\n    \"\"\"Detect horizontal grid lines from mask to calculate calibration.\"\"\"\n    # Sum across width to find horizontal lines\n    horizontal_projection = np.sum(grid_mask, axis=1)\n\n    if horizontal_projection.max() == 0:\n        return []\n\n    # Find peaks (rows with many grid pixels)\n    threshold = np.percentile(horizontal_projection, 95)\n    line_positions = np.where(horizontal_projection > threshold)[0]\n\n    # Group nearby positions\n    lines = []\n    if len(line_positions) > 0:\n        current_line = [line_positions[0]]\n        for pos in line_positions[1:]:\n            if pos - current_line[-1] > 5:  # New line\n                lines.append(int(np.mean(current_line)))\n                current_line = [pos]\n            else:\n                current_line.append(pos)\n        lines.append(int(np.mean(current_line)))\n\n    return lines\n\ndef predict_mask_with_calibration(model, image_tensor, device):\n    \"\"\"Run inference and get segmentation mask with calibration.\"\"\"\n    with torch.no_grad():\n        image_tensor = image_tensor.to(device)\n        output = model(image_tensor)\n\n        # Apply sigmoid for probabilities\n        probs = torch.sigmoid(output)\n\n        # Get binary mask (threshold at 0.5)\n        mask = (probs > 0.5).float()\n\n    mask_np = mask.cpu().numpy()[0]  # (2, H, W)\n\n    # CRITICAL: Detect calibration using grid mask\n    grid_mask = mask_np[1]  # Channel 1 = grid lines\n\n    # Find horizontal grid lines\n    horizontal_lines = detect_horizontal_lines(grid_mask)\n\n    if len(horizontal_lines) >= 2:\n        # Calculate spacing between grid lines\n        grid_spacing = np.median(np.diff(sorted(horizontal_lines)))\n\n        # ECG paper has TWO grid sizes:\n        # - Small grid: 1mm spacing (0.1 mV for standard 10mm/mV)\n        # - Large grid: 5mm spacing (0.5 mV)\n        # We want to calibrate to LARGE grid (5mm = 0.5 mV)\n\n        # For 512px height and ~20-25 grid lines:\n        # This is detecting SMALL grid (1mm spacing)\n        # So grid_spacing ≈ 12px = 0.1 mV → multiply by 10 to get 1 mV\n\n        # Standard ECG: 10mm = 1 mV\n        # If we detect 20+ lines in 512px, those are 1mm (small) grids\n        # 5mm (large grid) = 5 * grid_spacing\n\n        if len(horizontal_lines) > 15:\n            # Many lines detected → small grid (1mm)\n            # 5mm spacing = 5 * grid_spacing = 0.5 mV\n            # 10mm spacing = 10 * grid_spacing = 1.0 mV\n            pixels_per_mv = 10 * grid_spacing  # Convert small grid to 1mV\n            print(f\"Detected small grid: {grid_spacing:.1f}px spacing\")\n            print(f\"Calibration: {pixels_per_mv:.1f} pixels/mV\")\n        else:\n            # Few lines detected → large grid (5mm)\n            # Each spacing = 0.5 mV, so 1 mV = 2 * spacing\n            pixels_per_mv = 2 * grid_spacing\n            print(f\"Detected large grid: {grid_spacing:.1f}px spacing\")\n            print(f\"Calibration: {pixels_per_mv:.1f} pixels/mV\")\n\n    else:\n        # Fallback: Use empirical value for 512px images\n        # Original 1700px images have ~40-80 px/mV\n        # Scaled to 512px: 40-80 * (512/1700) ≈ 12-24 px/mV\n        # But we need to account for small grid → large grid conversion\n        pixels_per_mv = 120.0  # Conservative estimate (10x small grid)\n        print(f\"No grid lines detected, using default: {pixels_per_mv:.1f} pixels/mV\")\n\n    return mask_np, pixels_per_mv\n\n# ============================================================================\n# SIGNAL EXTRACTION WITH CALIBRATION\n# ============================================================================\n\ndef extract_signal_from_mask(mask, pixels_per_mv, target_length=5000, debug=False):\n    \"\"\"\n    Extract ECG signal from waveform mask with proper mV conversion.\n\n    Args:\n        mask: Binary mask (H, W) with waveform pixels\n        pixels_per_mv: Calibration factor (pixels per millivolt)\n        target_length: Target signal length for resampling\n        debug: If True, print debug information\n\n    Returns:\n        signal: 1D array of signal values in mV\n    \"\"\"\n    h, w = mask.shape\n    signal = []\n\n    # Find baseline (median y-position of all waveform pixels)\n    all_y_positions = []\n    for x in range(w):\n        column = mask[:, x]\n        y_positions = np.where(column > 0)[0]\n        if len(y_positions) > 0:\n            all_y_positions.extend(y_positions.tolist())\n\n    if len(all_y_positions) == 0:\n        if debug:\n            print(\"WARNING: No waveform pixels detected!\")\n        return np.zeros(target_length)\n\n    # Baseline is the median position (0 mV reference)\n    baseline = np.median(all_y_positions)\n\n    if debug:\n        print(f\"Baseline position: {baseline:.1f} pixels\")\n        print(f\"Total waveform pixels: {len(all_y_positions)}\")\n        print(f\"Calibration: {pixels_per_mv:.1f} pixels/mV\")\n\n    # For each column, find the y-position of the waveform\n    columns_with_signal = 0\n    for x in range(w):\n        column = mask[:, x]\n        y_positions = np.where(column > 0)[0]\n\n        if len(y_positions) > 0:\n            columns_with_signal += 1\n            # Use median y-position (robust to noise)\n            y_median = np.median(y_positions)\n\n            # CRITICAL: Convert pixel distance to mV\n            # Negative because y increases downward in images\n            pixel_distance = baseline - y_median\n            signal_value_mv = pixel_distance / pixels_per_mv\n\n            signal.append(signal_value_mv)\n        else:\n            # No waveform detected, use previous value or 0\n            signal.append(signal[-1] if signal else 0.0)\n\n    if debug:\n        print(f\"Columns with signal: {columns_with_signal}/{w}\")\n        if signal:\n            print(f\"Signal range before clipping: [{min(signal):.3f}, {max(signal):.3f}] mV\")\n\n    # Convert to numpy array\n    signal = np.array(signal)\n\n    # Remove DC offset (center around 0)\n    signal = signal - np.median(signal)\n\n    # Clip to physiologically reasonable range (±2 mV for normal ECG)\n    signal = np.clip(signal, -2.0, 2.0)\n\n    if debug:\n        print(f\"Signal range after processing: [{signal.min():.3f}, {signal.max():.3f}] mV\")\n        print(f\"Signal mean: {signal.mean():.3f}, std: {signal.std():.3f}\")\n\n    # Resample to target length\n    if len(signal) > 0:\n        x_old = np.linspace(0, 1, len(signal))\n        x_new = np.linspace(0, 1, target_length)\n        f = interp1d(x_old, signal, kind='linear', fill_value='extrapolate')\n        signal_resampled = f(x_new)\n    else:\n        signal_resampled = np.zeros(target_length)\n\n    return signal_resampled\n\n# ============================================================================\n# SUBMISSION GENERATION\n# ============================================================================\n\ndef generate_submission(test_dir, model, device, output_path='submission.csv', debug_first=True):\n    \"\"\"\n    Generate submission CSV with proper calibration.\n\n    Args:\n        test_dir: Path to test images directory\n        model: Trained U-Net model\n        device: torch device\n        output_path: Output CSV path\n        debug_first: If True, show detailed debug info for first image\n    \"\"\"\n    test_images = sorted(Path(test_dir).glob('*.png'))\n\n    submissions = []\n\n    print(f'Processing {len(test_images)} test images...')\n\n    for idx, img_path in enumerate(tqdm(test_images)):\n        image_id = img_path.stem\n        is_first = (idx == 0)\n\n        # Preprocess and predict WITH CALIBRATION\n        img_tensor = preprocess_image(img_path)\n        mask, pixels_per_mv = predict_mask_with_calibration(model, img_tensor, device)\n\n        if is_first and debug_first:\n            print(f\"\\n{'='*60}\")\n            print(f\"DEBUG INFO for first image: {image_id}\")\n            print(f\"{'='*60}\")\n            print(f\"Mask shape: {mask.shape}\")\n            print(f\"Channel 0 (waveform) pixels: {np.sum(mask[0] > 0)}\")\n            print(f\"Channel 1 (grid) pixels: {np.sum(mask[1] > 0)}\")\n            print(f\"Final calibration: {pixels_per_mv:.1f} pixels/mV\")\n\n        # Extract waveform mask (channel 0)\n        waveform_mask = mask[0]\n\n        # Extract signal with calibration\n        signal = extract_signal_from_mask(\n            waveform_mask,\n            pixels_per_mv,\n            target_length=5000,\n            debug=(is_first and debug_first)\n        )\n\n        # Create submission rows\n        lead_name = 'I'  # Single lead for now\n        for sample_idx, value in enumerate(signal):\n            row_id = f\"{image_id}_{sample_idx}_{lead_name}\"\n            submissions.append({'id': row_id, 'value': value})\n\n    # Save\n    df = pd.DataFrame(submissions)\n    df.to_csv(output_path, index=False)\n\n    print(f'\\n{\"=\"*60}')\n    print(f'Submission saved to {output_path}')\n    print(f'Total rows: {len(df)}')\n    print(f'\\n{\"=\"*60}')\n    print(f'Signal statistics:')\n    print(f'{\"=\"*60}')\n    stats = df['value'].describe()\n    print(stats)\n\n    # Quality checks\n    print(f'\\n{\"=\"*60}')\n    print(f'QUALITY CHECKS:')\n    print(f'{\"=\"*60}')\n\n    mean_val = stats['mean']\n    std_val = stats['std']\n    min_val = stats['min']\n    max_val = stats['max']\n\n    # Check 1: Mean should be near 0\n    if abs(mean_val) < 0.3:\n        print(f'✓ Mean value OK: {mean_val:.3f} (expected: ~0)')\n    else:\n        print(f'⚠️  Mean value high: {mean_val:.3f} (expected: ~0)')\n\n    # Check 2: Std should be 0.1-0.5 mV\n    if 0.05 < std_val < 0.8:\n        print(f'✓ Standard deviation OK: {std_val:.3f}')\n    else:\n        print(f'⚠️  Standard deviation unusual: {std_val:.3f} (expected: 0.1-0.5)')\n\n    # Check 3: Range should be ±1-2 mV\n    if -2.5 < min_val < -0.2 and 0.2 < max_val < 2.5:\n        print(f'✓ Signal range OK: [{min_val:.3f}, {max_val:.3f}]')\n    else:\n        print(f'⚠️  Signal range unusual: [{min_val:.3f}, {max_val:.3f}] (expected: ±0.5-2 mV)')\n\n    # Sanity check\n    unique_values = df['value'].nunique()\n    if unique_values < 100:\n        print(f'⚠️  WARNING: Only {unique_values} unique values detected!')\n    else:\n        print(f'✓ Good diversity: {unique_values} unique values')\n\n    print(f'\\nPreview (first 20 rows):')\n    print(df.head(20))\n\n    return df\n\n# =============================================================================\n# MAIN EXECUTION\n# =============================================================================\n\n# Generate submission with calibration\ntest_dir = '/kaggle/input/physionet-ecg-image-digitization/test'\nsubmission_df = generate_submission(test_dir, model, device, 'submission.csv', debug_first=True)\n\nprint('\\n' + '='*60)\nprint('SUBMISSION COMPLETE!')\nprint('='*60)\nprint('File: submission.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T13:51:56.435438Z","iopub.execute_input":"2025-11-01T13:51:56.435953Z","iopub.status.idle":"2025-11-01T13:51:56.865902Z","shell.execute_reply.started":"2025-11-01T13:51:56.435930Z","shell.execute_reply":"2025-11-01T13:51:56.865289Z"}},"outputs":[],"execution_count":null}]}