{"metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 4, "cells": [{"cell_type": "markdown", "metadata": {}, "source": "# PhysioNet ECG Digitization - Calibrated v10\n\nUsing the trained calibration approach that got 0.10 dB, with proper lead handling."}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom pathlib import Path\nfrom scipy.interpolate import interp1d\nfrom scipy.signal import savgol_filter\nfrom scipy.stats import linregress\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nDATA_DIR = Path('/kaggle/input/physionet-ecg-image-digitization')\nOUTPUT_DIR = Path('/kaggle/working')\n\n# Standard 12-lead layout for 2.5-sec strips (3 rows x 4 cols)\n# Lead II rhythm strip is handled separately\nLEAD_LAYOUT = {\n    'I': (0, 0), 'aVR': (1, 0), 'V1': (2, 0), 'V4': (3, 0),\n    'II': (0, 1), 'aVL': (1, 1), 'V2': (2, 1), 'V5': (3, 1),\n    'III': (0, 2), 'aVF': (1, 2), 'V3': (2, 2), 'V6': (3, 2),\n}\n\nprint('Setup complete')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def extract_lead_region(img, lead, n_samples):\n    \"\"\"Extract lead region based on number of samples needed.\"\"\"\n    h, w = img.shape[:2]\n    \n    # For 10000 samples (rhythm strip), use bottom row spanning full width\n    if n_samples == 10000:\n        row_h = h // 4\n        return img[3*row_h:, :]\n    \n    # For 2500 samples, use the standard grid position\n    if lead not in LEAD_LAYOUT:\n        return img[:h//4, :w//4]\n    \n    col, row = LEAD_LAYOUT[lead]\n    col_w = w // 4\n    row_h = h // 4\n    return img[row*row_h:(row+1)*row_h, col*col_w:(col+1)*col_w]\n\ndef detect_trace_signal(region):\n    \"\"\"Extract raw y-positions of trace from region.\"\"\"\n    if len(region.shape) == 3:\n        gray = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY)\n    else:\n        gray = region.copy()\n    \n    h, w = gray.shape\n    inverted = 255 - gray\n    \n    binary = cv2.adaptiveThreshold(\n        inverted, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\n        cv2.THRESH_BINARY, 21, 3\n    )\n    \n    signal = np.zeros(w)\n    valid = np.zeros(w, dtype=bool)\n    \n    for x in range(w):\n        col_mask = binary[:, x]\n        col_intensity = inverted[:, x]\n        trace_y = np.where(col_mask > 128)[0]\n        \n        if len(trace_y) > 0:\n            weights = col_intensity[trace_y].astype(float) + 1\n            y_center = np.average(trace_y, weights=weights)\n            signal[x] = (h/2 - y_center) / (h/2)\n            valid[x] = True\n    \n    return signal, valid\n\ndef resample_signal(signal, valid, target_len):\n    x_orig = np.arange(len(signal))\n    \n    if valid.sum() > 5:\n        try:\n            f = interp1d(x_orig[valid], signal[valid], kind='linear',\n                        fill_value='extrapolate', bounds_error=False)\n            signal = f(x_orig)\n        except:\n            pass\n    \n    if len(signal) > 11:\n        try:\n            signal = savgol_filter(signal, 11, 3)\n        except:\n            pass\n    \n    x_new = np.linspace(0, len(signal)-1, target_len)\n    try:\n        f = interp1d(np.arange(len(signal)), signal, kind='linear')\n        return f(x_new)\n    except:\n        return np.zeros(target_len)\n\nprint('Functions defined')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# TRAINING PHASE: Learn calibration from training data\nprint('=== TRAINING PHASE ===')\n\ntrain_csv = pd.read_csv(DATA_DIR / 'train.csv')\ntrain_ids = train_csv['id'].unique()\nprint(f'Training ECGs: {len(train_ids)}')\n\n# Collect calibration data per lead\ncalibration_data = {lead: {'raw': [], 'truth': []} for lead in LEAD_LAYOUT.keys()}\n\nfor ecg_id in tqdm(train_ids[:300], desc='Collecting calibration data'):\n    ecg_folder = DATA_DIR / 'train' / str(ecg_id)\n    if not ecg_folder.exists():\n        continue\n    \n    images = list(ecg_folder.glob('*.png'))\n    csv_files = list(ecg_folder.glob('*.csv'))\n    if not images or not csv_files:\n        continue\n    \n    img = cv2.imread(str(images[0]))\n    if img is None:\n        continue\n    \n    try:\n        gt_df = pd.read_csv(csv_files[0])\n    except:\n        continue\n    \n    ecg_meta = train_csv[train_csv['id'] == ecg_id]\n    if len(ecg_meta) == 0:\n        continue\n    fs = ecg_meta['fs'].iloc[0]\n    \n    for lead in LEAD_LAYOUT.keys():\n        if lead not in gt_df.columns:\n            continue\n        \n        # Use 2500 sample regions for calibration (standard strips)\n        region = extract_lead_region(img, lead, 2500)\n        raw_signal, valid = detect_trace_signal(region)\n        \n        samples_per_strip = int(2.5 * fs)\n        col, _ = LEAD_LAYOUT[lead]\n        start = col * samples_per_strip\n        end = min(start + samples_per_strip, len(gt_df))\n        \n        if end <= start:\n            continue\n        \n        gt_signal = gt_df[lead].values[start:end]\n        raw_resampled = resample_signal(raw_signal, valid, len(gt_signal))\n        \n        if len(raw_resampled) == len(gt_signal) and np.std(raw_resampled) > 0.01:\n            calibration_data[lead]['raw'].append(raw_resampled)\n            calibration_data[lead]['truth'].append(gt_signal)\n\nprint('Calibration data collected')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Fit calibration\nprint('\\n=== Fitting calibration ===')\ncalibration = {}\n\nfor lead in LEAD_LAYOUT.keys():\n    raw = calibration_data[lead]['raw']\n    truth = calibration_data[lead]['truth']\n    \n    if len(raw) < 5:\n        calibration[lead] = {'scale': 1.5, 'offset': 0.0}\n        continue\n    \n    X = np.concatenate(raw)\n    y = np.concatenate(truth)\n    mask = np.isfinite(X) & np.isfinite(y)\n    X, y = X[mask], y[mask]\n    \n    if len(X) < 100:\n        calibration[lead] = {'scale': 1.5, 'offset': 0.0}\n        continue\n    \n    try:\n        slope, intercept, r, _, _ = linregress(X, y)\n        calibration[lead] = {'scale': slope, 'offset': intercept}\n        print(f'{lead}: scale={slope:.3f}, offset={intercept:.4f}, R\u00b2={r**2:.3f}')\n    except:\n        calibration[lead] = {'scale': 1.5, 'offset': 0.0}\n\nprint('Calibration complete')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# INFERENCE PHASE\nprint('\\n=== INFERENCE PHASE ===')\n\ntest_meta = pd.read_csv(DATA_DIR / 'test.csv')\nprint(f'Test ECGs: {test_meta[\"id\"].nunique()}')\nprint(f'Total rows needed: {test_meta[\"number_of_rows\"].sum()}')\n\nall_rows = []\n\nfor ecg_id in tqdm(test_meta['id'].unique(), desc='Processing test ECGs'):\n    image_path = DATA_DIR / 'test' / f'{ecg_id}.png'\n    ecg_meta = test_meta[test_meta['id'] == ecg_id]\n    \n    if not image_path.exists():\n        for _, row in ecg_meta.iterrows():\n            for i in range(row['number_of_rows']):\n                all_rows.append({'id': f\"{ecg_id}_{i}_{row['lead']}\", 'value': 0.0})\n        continue\n    \n    img = cv2.imread(str(image_path))\n    if img is None:\n        for _, row in ecg_meta.iterrows():\n            for i in range(row['number_of_rows']):\n                all_rows.append({'id': f\"{ecg_id}_{i}_{row['lead']}\", 'value': 0.0})\n        continue\n    \n    for _, row in ecg_meta.iterrows():\n        lead = row['lead']\n        n_samples = row['number_of_rows']\n        \n        try:\n            region = extract_lead_region(img, lead, n_samples)\n            raw_signal, valid = detect_trace_signal(region)\n            resampled = resample_signal(raw_signal, valid, n_samples)\n            \n            # Apply calibration\n            cal = calibration.get(lead, {'scale': 1.5, 'offset': 0.0})\n            signal = resampled * cal['scale'] + cal['offset']\n            signal = signal - np.median(signal)  # Baseline correction\n            \n            for i in range(n_samples):\n                val = float(signal[i]) if i < len(signal) else 0.0\n                val = np.clip(val, -5.0, 5.0)\n                if not np.isfinite(val):\n                    val = 0.0\n                all_rows.append({'id': f'{ecg_id}_{i}_{lead}', 'value': val})\n        except:\n            for i in range(n_samples):\n                all_rows.append({'id': f'{ecg_id}_{i}_{lead}', 'value': 0.0})\n\nprint(f'Generated {len(all_rows)} predictions')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Create submission\nsubmission = pd.DataFrame(all_rows)\nprint(f'\\nSubmission shape: {submission.shape}')\nprint(f\"Value range: [{submission['value'].min():.4f}, {submission['value'].max():.4f}]\")\n\n# Verify against sample\nsample = pd.read_parquet(DATA_DIR / 'sample_submission.parquet')\nprint(f'\\nSample shape: {sample.shape}')\nprint(f'Match: {len(submission) == len(sample)}')\n\n# Check ID match\nour_ids = set(submission['id'])\nsample_ids = set(sample['id'])\nprint(f'ID match: {our_ids == sample_ids}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Save\nsubmission.to_parquet('/kaggle/working/submission.parquet', index=False)\nsubmission.to_csv('/kaggle/working/submission.csv', index=False)\n\nprint('=== OUTPUT FILES ===')\nfor f in os.listdir('/kaggle/working'):\n    print(f'  {f}: {os.path.getsize(f\"/kaggle/working/{f}\"):,} bytes')\n\nprint('\\nDone!')"}]}