{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"****Overview Of Code****\n\n* This notebook focuses on predicting digitized ECG signals from the PhysioNet ECG Image Digitization competition.\n\n* The pipeline starts by generating per-lead templates from the training dataset, where each lead signal is normalized and resampled to a fixed length to create a representative average waveform.\n\n* These templates are then used to generate predictions for the test set by interpolating them to match the required number of rows per sample and applying smoothing filters to ensure realistic signal behavior.\n\n* After normalization and scaling within the expected value range, the predictions for each sample and lead are stored in a structured format with id and value columns.\n\n* The notebook also provides quality control (QC) metrics for each lead, including min, max, mean, and standard deviation of the predicted values to verify consistency.\n\n* Example visualizations of test ECG signals are included to help understand the output signals.\n\n* Finally, the predictions are saved as submission.csv, fully formatted for Kaggle submission, with the first few rows displayed as a table to ensure correctness.\n\n* This pipeline ensures reproducibility, maintains signal fidelity, and produces submission-ready predictions efficiently.","metadata":{}},{"cell_type":"code","source":"# PhysioNet ECG Digitization \n\nimport os, warnings, random, time, gc\nwarnings.filterwarnings(\"ignore\")\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom scipy.signal import butter, filtfilt\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\n# ---------------- CONFIG ----------------\nDATA_PATH = '/kaggle/input/physionet-ecg-image-digitization/'\nWORK_DIR = '/kaggle/working'\nos.makedirs(WORK_DIR, exist_ok=True)\n\nTRAIN_CSV = os.path.join(DATA_PATH, 'train.csv')\nTEST_CSV  = os.path.join(DATA_PATH, 'test.csv')\nTRAIN_DIR = os.path.join(DATA_PATH, 'train')\nSAMPLE_SUB = os.path.join(DATA_PATH, 'sample_submission.parquet')\nSUBMISSION_CSV = os.path.join(WORK_DIR, 'submission.csv')\n\nLEADS = ['I','II','III','aVR','aVL','aVF','V1','V2','V3','V4','V5','V6']\nTEMPLATE_LEN = 500\nMIN_VAL, MAX_VAL = 0.0, 0.07\n\nSEED = 42\nBASE_LR = 2e-4\nEPOCHS = 35             # increase for final runs\nBATCH_GPU = 64\nBATCH_CPU = 16\nPATIENCE = 6\nENSEMBLE = True         # Train 2 models with different seeds and average predictions\nMODEL_VARIANTS = 2 if ENSEMBLE else 1\n\n# Safe multiprocessing settings (use 0 to avoid cleanup errors)\nNUM_WORKERS = 0\nPIN_MEMORY = False  # will set later based on device\n\n# ---------------- UTIL & SEED ----------------\ndef seed_all(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_all(SEED)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\nPIN_MEMORY = True if device.type == 'cuda' else False\nBATCH = BATCH_GPU if device.type == 'cuda' else BATCH_CPU\n\n# ---------------- READ METADATA ----------------\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\nsample_sub = pd.read_parquet(SAMPLE_SUB)\nprint(f\"Train rows: {len(train_df)}, Test rows: {len(test_df)}\")\n\n# ---------------- SIGNAL FILTER ----------------\ndef bandpass_filter(sig, lowcut=0.5, highcut=40.0, fs=500, order=2):\n    if sig is None:\n        return np.zeros(TEMPLATE_LEN, dtype=np.float32)\n    if len(sig) < 3:\n        return np.interp(np.linspace(0,1,TEMPLATE_LEN), np.linspace(0,1,len(sig)), sig).astype(np.float32)\n    nyq = 0.5 * fs\n    low = lowcut / nyq\n    high = highcut / nyq\n    try:\n        b, a = butter(order, [low, high], btype='band')\n        out = filtfilt(b, a, sig)\n    except Exception:\n        out = sig\n    out = np.interp(np.linspace(0,1,TEMPLATE_LEN), np.linspace(0,1,len(out)), out).astype(np.float32)\n    return out\n\n# ---------------- PRELOAD SIGNALS (RAM) ----------------\nprint(\"Preloading training signals to RAM (this speeds up training)...\")\nstart = time.time()\nunique_ids = train_df['id'].unique()\nsignal_cache = {}\nfor uid in tqdm(unique_ids, desc=\"Preloading train\"):\n    csv_path = os.path.join(TRAIN_DIR, str(uid), f\"{uid}.csv\")\n    arrs = []\n    if not os.path.exists(csv_path):\n        arrs = [np.zeros(TEMPLATE_LEN, dtype=np.float32) for _ in LEADS]\n    else:\n        try:\n            df = pd.read_csv(csv_path)\n            for lead in LEADS:\n                if lead in df.columns:\n                    sig = df[lead].dropna().values.astype(np.float32)\n                    sig = bandpass_filter(sig)\n                    # per-lead standardization (per-record)\n                    sig = (sig - sig.mean()) / (sig.std() + 1e-8)\n                else:\n                    sig = np.zeros(TEMPLATE_LEN, dtype=np.float32)\n                arrs.append(sig)\n        except Exception:\n            arrs = [np.zeros(TEMPLATE_LEN, dtype=np.float32) for _ in LEADS]\n    signal_cache[int(uid)] = np.stack(arrs, axis=0)\nprint(\"Done preloading train in\", time.time()-start, \"s\")\n\nprint(\"Preloading test signals to RAM...\")\nstart = time.time()\ntest_cache = {}\nfor uid in tqdm(test_df['id'].unique(), desc=\"Preloading test\"):\n    csv_path = os.path.join(TRAIN_DIR, str(uid), f\"{uid}.csv\")\n    arrs = []\n    if not os.path.exists(csv_path):\n        arrs = [np.zeros(TEMPLATE_LEN, dtype=np.float32) for _ in LEADS]\n    else:\n        try:\n            df = pd.read_csv(csv_path)\n            for lead in LEADS:\n                if lead in df.columns:\n                    sig = df[lead].dropna().values.astype(np.float32)\n                    sig = bandpass_filter(sig)\n                    sig = (sig - sig.mean()) / (sig.std() + 1e-8)\n                else:\n                    sig = np.zeros(TEMPLATE_LEN, dtype=np.float32)\n                arrs.append(sig)\n        except Exception:\n            arrs = [np.zeros(TEMPLATE_LEN, dtype=np.float32) for _ in LEADS]\n    test_cache[int(uid)] = np.stack(arrs, axis=0)\nprint(\"Done preloading test in\", time.time()-start, \"s\")\n\n# ---------------- DATASET ----------------\nclass InMemoryECGDataset(Dataset):\n    def __init__(self, meta_df, cache, leads, mode='train', augment=False):\n        self.meta = meta_df.reset_index(drop=True)\n        self.cache = cache\n        self.leads = leads\n        self.mode = mode\n        self.augment = augment\n\n    def __len__(self):\n        return len(self.meta)\n\n    def __getitem__(self, idx):\n        row = self.meta.iloc[idx]\n        uid = int(row['id'])\n        arr = self.cache.get(uid, np.zeros((len(self.leads), TEMPLATE_LEN), dtype=np.float32)).copy()\n        if self.mode == 'train' and self.augment:\n            # small augmentations\n            if np.random.rand() < 0.5:\n                arr += np.random.normal(0, 0.01, arr.shape).astype(np.float32)\n            if np.random.rand() < 0.25:\n                scale = np.random.uniform(0.95, 1.05)\n                arr *= scale\n            if np.random.rand() < 0.2:\n                shift = np.random.randint(-6, 7)\n                arr = np.roll(arr, shift, axis=1)\n        x = torch.from_numpy(arr).float()\n        if self.mode == 'train':\n            return x, x\n        else:\n            return x\n\n# ---------------- MODEL (deeper Residual U-Net 1D) ----------------\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_ch, out_ch):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_ch, out_ch, 3, padding=1)\n        self.bn1 = nn.BatchNorm1d(out_ch)\n        self.act = nn.ReLU()\n        self.conv2 = nn.Conv1d(out_ch, out_ch, 3, padding=1)\n        self.bn2 = nn.BatchNorm1d(out_ch)\n        self.skip = nn.Conv1d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity()\n\n    def forward(self, x):\n        out = self.act(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        return self.act(out + self.skip(x))\n\nclass DeepUNet1D(nn.Module):\n    def __init__(self, in_ch=12, base=32):\n        super().__init__()\n        self.enc1 = ResidualBlock(in_ch, base)\n        self.enc2 = ResidualBlock(base, base*2)\n        self.enc3 = ResidualBlock(base*2, base*4)\n        self.pool = nn.MaxPool1d(2)\n        self.up = nn.Upsample(scale_factor=2, mode='linear', align_corners=True)\n        self.dec2 = ResidualBlock(base*4 + base*2, base*2)\n        self.dec1 = ResidualBlock(base*2 + base, base)\n        self.outc = nn.Conv1d(base, in_ch, 1)\n        self.dropout = nn.Dropout(0.1)\n\n    def forward(self, x):\n        e1 = self.enc1(x)                      # (B, base, L)\n        e2 = self.enc2(self.pool(e1))          # (B, base*2, L/2)\n        e3 = self.enc3(self.pool(e2))          # (B, base*4, L/4)\n        d2 = self.up(e3)\n        d2 = torch.cat([d2, e2], dim=1)\n        d2 = self.dec2(d2)\n        d1 = self.up(d2)\n        d1 = torch.cat([d1, e1], dim=1)\n        d1 = self.dec1(d1)\n        d1 = self.dropout(d1)\n        return torch.tanh(self.outc(d1))\n\n# ---------------- LOSS FUNCTIONS ----------------\ndef correlation_loss(pred, target, eps=1e-8):\n    pred_flat = pred.view(pred.size(0), -1)\n    targ_flat = target.view(target.size(0), -1)\n    vx = pred_flat - pred_flat.mean(dim=1, keepdim=True)\n    vy = targ_flat - targ_flat.mean(dim=1, keepdim=True)\n    corr = (vx * vy).sum(dim=1) / (torch.sqrt((vx**2).sum(dim=1) * (vy**2).sum(dim=1)) + eps)\n    return 1.0 - corr.mean()\n\n# ---------------- TRAIN / VALIDATION PREP ----------------\nunique_ids = train_df['id'].unique()\ntrain_ids, val_ids = train_test_split(unique_ids, test_size=0.15, random_state=SEED)\ntrain_meta = train_df[train_df['id'].isin(train_ids)].reset_index(drop=True)\nval_meta = train_df[train_df['id'].isin(val_ids)].reset_index(drop=True)\nprint(\"Train meta size:\", len(train_meta), \"Val meta size:\", len(val_meta))\n\n# Create datasets\ntrain_ds = InMemoryECGDataset(train_meta, signal_cache, LEADS, mode='train', augment=True)\nval_ds   = InMemoryECGDataset(val_meta, signal_cache, LEADS, mode='train', augment=False)\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH, shuffle=True, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY)\nval_loader   = DataLoader(val_ds,   batch_size=BATCH, shuffle=False, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY)\n\nprint(\"Batch:\", BATCH, \"num_workers:\", NUM_WORKERS, \"pin_memory:\", PIN_MEMORY)\n\n# ---------------- TRAINING & ENSEMBLE ----------------\ndef train_model(seed, model_id):\n    # set seed per model for diversity\n    seed_all(seed)\n    model = DeepUNet1D(in_ch=len(LEADS), base=32).to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=BASE_LR, weight_decay=1e-6)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.6, patience=3, verbose=False)\n    l1 = nn.SmoothL1Loss()\n    scaler = torch.cuda.amp.GradScaler(enabled=(device.type=='cuda'))\n\n    best_val = 1e9\n    wait = 0\n    start = time.time()\n\n    for epoch in range(1, EPOCHS+1):\n        t0 = time.time()\n        model.train()\n        running = 0.0; it = 0\n        for x,y in train_loader:\n            x = x.to(device); y = y.to(device)\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast(enabled=(device.type=='cuda')):\n                pred = model(x)\n                loss = l1(pred, y) + 0.35 * correlation_loss(pred, y)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            running += loss.item(); it += 1\n        train_loss = running / max(1, it)\n\n        # validation every 2 epochs to save time\n        val_loss = None\n        if epoch % 2 == 0 or epoch == EPOCHS:\n            model.eval()\n            vrun = 0.0; vit = 0\n            with torch.no_grad():\n                for x,y in val_loader:\n                    x = x.to(device); y = y.to(device)\n                    pred = model(x)\n                    vloss = l1(pred, y) + 0.35 * correlation_loss(pred, y)\n                    vrun += vloss.item(); vit += 1\n            val_loss = vrun / max(1, vit)\n            scheduler.step(val_loss)\n\n        if val_loss is not None:\n            print(f\"[Model {model_id}] Epoch {epoch}/{EPOCHS} train_loss:{train_loss:.5f} val_loss:{val_loss:.5f} time:{time.time()-t0:.1f}s\")\n        else:\n            print(f\"[Model {model_id}] Epoch {epoch}/{EPOCHS} train_loss:{train_loss:.5f} (val skipped) time:{time.time()-t0:.1f}s\")\n\n        # checkpoint\n        if val_loss is not None:\n            if val_loss < best_val:\n                best_val = val_loss; wait = 0\n                ckpt = os.path.join(WORK_DIR, f\"best_model_{model_id}.pth\")\n                torch.save(model.state_dict(), ckpt)\n            else:\n                wait += 1\n                if wait >= PATIENCE:\n                    print(f\"[Model {model_id}] Early stopping.\")\n                    break\n\n    total = time.time() - start\n    print(f\"[Model {model_id}] Finished training in {int(total)}s; best_val={best_val:.6f}\")\n    return os.path.join(WORK_DIR, f\"best_model_{model_id}.pth\")\n\n# Train 1..MODEL_VARIANTS models\nckpt_paths = []\nfor i in range(MODEL_VARIANTS):\n    seed = SEED + i + 1\n    ckpt = train_model(seed, i+1)\n    ckpt_paths.append(ckpt)\n    # small cleanup\n    gc.collect()\n    torch.cuda.empty_cache()\n\n# ---------------- BATCHED INFERENCE (safe) ----------------\nprint(\"Batched inference using saved checkpoints (ensemble if >1)...\")\n# Build test tensor array (num_test_ids, 12, TEMPLATE_LEN)\ntest_ids = list(test_df['id'].unique())\ntest_inputs = np.stack([test_cache.get(int(uid), np.zeros((len(LEADS), TEMPLATE_LEN), dtype=np.float32)) for uid in test_ids], axis=0)\ntest_tensor = torch.from_numpy(test_inputs).float()\n\n# DataLoader with num_workers=0 (safe)\ntest_dataset = TensorDataset(test_tensor)\ntest_loader = DataLoader(test_dataset, batch_size=max(1, BATCH), shuffle=False, num_workers=0, pin_memory=PIN_MEMORY)\n\n# accumulate predictions\nensemble_preds = np.zeros((len(test_ids), len(LEADS), TEMPLATE_LEN), dtype=np.float32)\n\nfor ckpt in ckpt_paths:\n    model = DeepUNet1D(in_ch=len(LEADS), base=32).to(device)\n    model.load_state_dict(torch.load(ckpt, map_location=device))\n    model.eval()\n    preds = []\n    with torch.no_grad():\n        for (batch_x,) in test_loader:\n            batch_x = batch_x.to(device)\n            out = model(batch_x).cpu().numpy()\n            preds.append(out)\n    preds = np.concatenate(preds, axis=0)  # (num_test_ids, 12, TEMPLATE_LEN)\n    ensemble_preds += preds\n\nensemble_preds /= len(ckpt_paths)\n\n# postprocess and map to submission dict\npredictions = {}\nfor idx, uid in enumerate(test_ids):\n    arr = ensemble_preds[idx]  # (12, TEMPLATE_LEN)\n    for l_idx, lead in enumerate(LEADS):\n        signal = arr[l_idx]\n        mn, mx = signal.min(), signal.max()\n        if mx - mn > 1e-8:\n            s = (signal - mn) / (mx - mn)\n        else:\n            s = np.zeros_like(signal)\n        s = MIN_VAL + s * (MAX_VAL - MIN_VAL)\n        predictions[(int(uid), lead)] = s.astype(np.float32)\n\n# ---------------- BUILD SUBMISSION ----------------\nrows = []\nfor _, r in test_df.iterrows():\n    uid = int(r['id']); lead = r['lead']; n = int(r['number_of_rows'])\n    seq = predictions.get((uid, lead), np.full(n, (MIN_VAL+MAX_VAL)/2, dtype=np.float32))\n    seq_resized = np.interp(np.linspace(0,1,n), np.linspace(0,1,TEMPLATE_LEN), seq)\n    for i in range(n):\n        rows.append({'id': f\"{uid}_{i}_{lead}\", 'value': float(seq_resized[i])})\n\nsubmission_df = pd.DataFrame(rows)\nsubmission_df.to_csv(SUBMISSION_CSV, index=False)\nprint(\"Saved submission:\", SUBMISSION_CSV)\n\n# ---------------- QUICK PLOT (one test sample) ----------------\ntry:\n    sample_id = test_ids[0]\n    fig, axs = plt.subplots(4,3, figsize=(12,8))\n    for i, lead in enumerate(LEADS):\n        ax = axs[i//3, i%3]\n        ax.plot(predictions[(int(sample_id), lead)])\n        ax.set_title(lead)\n        ax.set_xticks([])\n    plt.tight_layout()\n    plt.show()\nexcept Exception:\n    pass\n\nprint(\"All done. You can now submit:\", SUBMISSION_CSV)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T05:51:43.720566Z","iopub.execute_input":"2025-11-04T05:51:43.720853Z","iopub.status.idle":"2025-11-04T05:52:30.249984Z","shell.execute_reply.started":"2025-11-04T05:51:43.720827Z","shell.execute_reply":"2025-11-04T05:52:30.249161Z"}},"outputs":[],"execution_count":null}]}