{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"c5010f96","cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch","metadata":{},"outputs":[],"execution_count":null},{"id":"f56d2fca","cell_type":"code","source":"# Tree model setup: CatBoost + group split\nfrom catboost import CatBoostRegressor, Pool\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.preprocessing import StandardScaler\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"0c420a1e","cell_type":"markdown","source":"# Config","metadata":{}},{"id":"6266bb80","cell_type":"code","source":"class Config:\n    BASE_DIR = '/kaggle/input/physionet-ecg-image-digitization'\n    ITERATIONS = 5000\n    SEED = 42\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    # First run on a limited subset for speed; then we will train on full data\n    LIMIT_IDS = None           # set to None to use all ids immediately\n    SUBSAMPLE_EVERY = 2        # take every k-th sample within each series\n    # FULL_SUBSAMPLE_EVERY = 1   # heavier set; adjust if memory is tight\nrng = np.random.RandomState(Config.SEED)","metadata":{},"outputs":[],"execution_count":null},{"id":"b5cdaf27","cell_type":"code","source":"# Use config BASE_DIR\ntrain = pd.read_csv(Config.BASE_DIR + '/train.csv')\ntest = pd.read_csv(Config.BASE_DIR + '/test.csv')\nsubmission = pd.read_parquet(Config.BASE_DIR + '/sample_submission.parquet')","metadata":{},"outputs":[],"execution_count":null},{"id":"ea035d8c","cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nTRAIN_DIR = Config.BASE_DIR + '/train/'\nLEADS = ['I','II','III','aVR','aVL','aVF','V1','V2','V3','V4','V5','V6']","metadata":{},"outputs":[],"execution_count":null},{"id":"5ac43724","cell_type":"markdown","source":"# Metric ","metadata":{}},{"id":"37d756a7","cell_type":"code","source":"from typing import Tuple\n\nimport numpy as np\nimport pandas as pd\n\nimport scipy.optimize\nimport scipy.signal\n\n\nLEADS = ['I', 'II', 'III', 'aVR', 'aVL', 'aVF', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6']\nMAX_TIME_SHIFT = 0.2\nPERFECT_SCORE = 384\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef compute_power(label: np.ndarray, prediction: np.ndarray) -> Tuple[float, float]:\n    if label.ndim != 1 or prediction.ndim != 1:\n        raise ParticipantVisibleError('Inputs must be 1-dimensional arrays.')\n    finite_mask = np.isfinite(prediction)\n    if not np.any(finite_mask):\n        raise ParticipantVisibleError(\"The 'prediction' array contains no finite values (all NaN or inf).\")\n\n    prediction[~np.isfinite(prediction)] = 0\n    noise = label - prediction\n    p_signal = np.sum(label**2)\n    p_noise = np.sum(noise**2)\n    return p_signal, p_noise\n\n\ndef compute_snr(signal: float, noise: float) -> float:\n    if noise == 0:\n        # Perfect reconstruction\n        snr = PERFECT_SCORE\n    elif signal == 0:\n        snr = 0\n    else:\n        snr = min((signal / noise), PERFECT_SCORE)\n    return snr\n\n\ndef align_signals(label: np.ndarray, pred: np.ndarray, max_shift: float = float('inf')) -> np.ndarray:\n    if np.any(~np.isfinite(label)):\n        raise ParticipantVisibleError('values in label should all be finite')\n    if np.sum(np.isfinite(pred)) == 0:\n        raise ParticipantVisibleError('prediction can not all be infinite')\n\n    # Initialize the reference and digitized signals.\n    label_arr = np.asarray(label, dtype=np.float64)\n    pred_arr = np.asarray(pred, dtype=np.float64)\n\n    label_mean = np.mean(label_arr)\n    pred_mean = np.mean(pred_arr)\n\n    label_arr_centered = label_arr - label_mean\n    pred_arr_centered = pred_arr - pred_mean\n\n    # Compute the correlation between the reference and digitized signals and locate the maximum correlation.\n    correlation = scipy.signal.correlate(label_arr_centered, pred_arr_centered, mode='full')\n\n    n_label = np.size(label_arr)\n    n_pred = np.size(pred_arr)\n\n    lags = scipy.signal.correlation_lags(n_label, n_pred, mode='full')\n    valid_lags_mask = (lags >= -max_shift) & (lags <= max_shift)\n\n    max_correlation = np.nanmax(correlation[valid_lags_mask])\n    all_max_indices = np.flatnonzero(correlation == max_correlation)\n    best_idx = min(all_max_indices, key=lambda i: abs(lags[i]))\n    time_shift = lags[best_idx]\n    start_padding_len = max(time_shift, 0)\n    pred_slice_start = max(-time_shift, 0)\n    pred_slice_end = min(n_label - time_shift, n_pred)\n    end_padding_len = max(n_label - n_pred - time_shift, 0)\n    aligned_pred = np.concatenate((np.full(start_padding_len, np.nan), pred_arr[pred_slice_start:pred_slice_end], np.full(end_padding_len, np.nan)))\n\n    def objective_func(v_shift):\n        return np.nansum((label_arr - (aligned_pred - v_shift)) ** 2)\n\n    if np.any(np.isfinite(label_arr) & np.isfinite(aligned_pred)):\n        results = scipy.optimize.minimize_scalar(objective_func, method='Brent')\n        vertical_shift = results.x\n        aligned_pred -= vertical_shift\n    return aligned_pred\n\n\ndef _calculate_image_score(group: pd.DataFrame) -> float:\n    \"\"\"Helper function to calculate the total SNR score for a single image group.\"\"\"\n\n    unique_fs_values = group['fs'].unique()\n    if len(unique_fs_values) != 1:\n        raise ParticipantVisibleError('Sampling frequency should be consistent across each ecg')\n    sampling_frequency = unique_fs_values[0]\n    if sampling_frequency != int(len(group[group['lead'] == 'II']) / 10):\n        raise ParticipantVisibleError('The sequence_length should be sampling frequency * 10s')\n    sum_signal = 0\n    sum_noise = 0\n    for lead in LEADS:\n        sub = group[group['lead'] == lead]\n        label = sub['value_true'].values\n        pred = sub['value_pred'].values\n\n        aligned_pred = align_signals(label, pred, int(sampling_frequency * MAX_TIME_SHIFT))\n        p_signal, p_noise = compute_power(label, aligned_pred)\n        sum_signal += p_signal\n        sum_noise += p_noise\n    return compute_snr(sum_signal, sum_noise)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str) -> float:\n    \"\"\"\n    Compute the mean Signal-to-Noise Ratio (SNR) across multiple ECG leads and images for the PhysioNet 2025 competition.\n    The final score is the average of the sum of SNRs over different lines, averaged over all unique images.\n    Args:\n        solution: DataFrame with ground truth values. Expected columns: 'id' and one for each lead.\n        submission: DataFrame with predicted values. Expected columns: 'id' and one for each lead.\n        row_id_column_name: The name of the unique identifier column, typically 'id'.\n    Returns:\n        The final competition score.\n\n    Examples\n    --------\n    >>> import pandas as pd\n    >>> import numpy as np\n    >>> row_id_column_name = \"id\"\n    >>> solution = pd.DataFrame({'id': ['343_0_I', '343_1_I', '343_2_I', '343_0_III', '343_1_III','343_2_III','343_0_aVR', '343_1_aVR','343_2_aVR',\\\n    '343_0_aVL', '343_1_aVL', '343_2_aVL', '343_0_aVF', '343_1_aVF','343_2_aVF','343_0_V1', '343_1_V1', '343_2_V1','343_0_V2', '343_1_V2','343_2_V2',\\\n    '343_0_V3', '343_1_V3', '343_2_V3','343_0_V4', '343_1_V4', '343_2_V4', '343_0_V5', '343_1_V5','343_2_V5','343_0_V6', '343_1_V6','343_2_V6',\\\n    '343_0_II', '343_1_II','343_2_II', '343_3_II', '343_4_II', '343_5_II','343_6_II', '343_7_II','343_8_II','343_9_II','343_10_II','343_11_II'],\\\n    'fs': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\\\n    'value':[0.1,0.3,0.4,0.6,0.6,0.4,0.2,0.3,0.4,0.5,0.2,0.7,0.2,0.3,0.4,0.8,0.6,0.7, 0.2,0.3,-0.1,0.5,0.6,0.7,0.2,0.9,0.4,0.5,0.6,0.7,0.1,0.3,0.4,\\\n    0.6,0.6,0.4,0.2,0.3,0.4,0.5,0.2,0.7,0.2,0.3,0.4]})\n    >>> submission = solution.copy()\n    >>> round(score(solution, submission, row_id_column_name), 4)\n    25.8433\n    >>> submission.loc[0, 'value'] = 0.9 # Introduce some noise\n    >>> round(score(solution, submission, row_id_column_name), 4)\n    13.6291\n    >>> submission.loc[4, 'value'] = 0.3 # Introduce some noise\n    >>> round(score(solution, submission, row_id_column_name), 4)\n    13.0576\n\n    >>> solution = pd.DataFrame({'id': ['343_0_I', '343_1_I', '343_2_I', '343_0_III', '343_1_III','343_2_III','343_0_aVR', '343_1_aVR','343_2_aVR',\\\n    '343_0_aVL', '343_1_aVL', '343_2_aVL', '343_0_aVF', '343_1_aVF','343_2_aVF','343_0_V1', '343_1_V1', '343_2_V1','343_0_V2', '343_1_V2','343_2_V2',\\\n    '343_0_V3', '343_1_V3', '343_2_V3','343_0_V4', '343_1_V4', '343_2_V4', '343_0_V5', '343_1_V5','343_2_V5','343_0_V6', '343_1_V6','343_2_V6',\\\n    '343_0_II', '343_1_II','343_2_II', '343_3_II', '343_4_II', '343_5_II','343_6_II', '343_7_II','343_8_II','343_9_II','343_10_II','343_11_II'],\\\n    'fs': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\\\n    'value':[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]})\n    >>> round(score(solution, submission, row_id_column_name), 4)\n    -384\n    >>> submission = solution.copy()\n    >>> round(score(solution, submission, row_id_column_name), 4)\n    25.8433\n\n    >>> # test alignment\n    >>> label = np.array([0, 1, 2, 1, 0])\n    >>> pred = np.array([0, 1, 2, 1, 0])\n    >>> aligned = align_signals(label, pred)\n    >>> expected_array = np.array([0, 1, 2, 1, 0])\n    >>> np.allclose(aligned, expected_array, equal_nan=True)\n    True\n\n    >>> # Test 2: Vertical shift (DC offset) should be removed\n    >>> label = np.array([0, 1, 2, 1, 0])\n    >>> pred = np.array([10, 11, 12, 11, 10])\n    >>> aligned = align_signals(label, pred)\n    >>> expected_array = np.array([0, 1, 2, 1, 0])\n    >>> np.allclose(aligned, expected_array, equal_nan=True)\n    True\n\n    >>> # Test 3: Time shift should be corrected\n    >>> label = np.array([0, 0, 1, 2, 1, 0., 0.])\n    >>> pred = np.array([1, 2, 1, 0, 0, 0, 0])\n    >>> aligned = align_signals(label, pred)\n    >>> expected_array = np.array([np.nan, np.nan, 1, 2, 1, 0, 0])\n    >>> np.allclose(aligned, expected_array, equal_nan=True)\n    True\n    \n    >>> # Test 4: max_shift constraint prevents optimal alignment\n    >>> label = np.array([0, 0, 0, 0, 1, 2, 1]) # Peak is far\n    >>> pred = np.array([1, 2, 1, 0, 0, 0, 0])\n    >>> aligned = align_signals(label, pred, max_shift=10)\n    >>> expected_array = np.array([ np.nan, np.nan, np.nan, np.nan, 1, 2, 1])\n    >>> np.allclose(aligned, expected_array, equal_nan=True)\n    True\n\n    \"\"\"\n    for df in [solution, submission]:\n        if row_id_column_name not in df.columns:\n            raise ParticipantVisibleError(f\"'{row_id_column_name}' column not found in DataFrame.\")\n        if df['value'].isna().any():\n            raise ParticipantVisibleError('NaN exists in solution/submission')\n        if not np.isfinite(df['value']).all():\n            raise ParticipantVisibleError('Infinity exists in solution/submission')\n\n    submission = submission[['id', 'value']]\n    merged_df = pd.merge(solution, submission, on=row_id_column_name, suffixes=('_true', '_pred'))\n    merged_df['image_id'] = merged_df[row_id_column_name].str.split('_').str[0]\n    merged_df['row_id'] = merged_df[row_id_column_name].str.split('_').str[1].astype('int64')\n    merged_df['lead'] = merged_df[row_id_column_name].str.split('_').str[2]\n    merged_df.sort_values(by=['image_id', 'row_id', 'lead'], inplace=True)\n    image_scores = merged_df.groupby('image_id').apply(_calculate_image_score, include_groups=False)\n    return max(float(10 * np.log10(image_scores.mean())), -PERFECT_SCORE)","metadata":{},"outputs":[],"execution_count":null},{"id":"e4770fb4","cell_type":"markdown","source":"# Prepare features","metadata":{}},{"id":"89b4eb94","cell_type":"code","source":"# Map id -> fs from train.csv\nFS_MAP = dict(zip(train['id'].astype(str), train['fs'].astype(int)))","metadata":{},"outputs":[],"execution_count":null},{"id":"82138cbf","cell_type":"code","source":"def iter_windows(sig: np.ndarray, fs: int, lead: str, crop_mode: str = 'first',\n                 step_sec_ii: float = 2.0, step_sec_other: float = 0.5):\n    \"\"\"Yield fixed-length windows across the full signal (overlapping).\"\"\"\n    win_len = int(round(fs * (10.0 if lead == 'II' else 2.5)))\n    if len(sig) <= win_len:\n        yield crop_to_expected_length(sig, fs, lead, mode=crop_mode)\n        return\n    step = int(round(fs * (step_sec_ii if lead == 'II' else step_sec_other)))\n    step = max(1, min(step, win_len))\n    last_start = len(sig) - win_len\n    for start in range(0, last_start + 1, step):\n        yield sig[start:start + win_len]\n    # ensure the tail is covered\n    if (last_start % step) != 0:\n        yield sig[-win_len:]","metadata":{},"outputs":[],"execution_count":null},{"id":"e38db62e","cell_type":"code","source":"def infer_fs_from_df(df: pd.DataFrame) -> int:\n    if 'II' in df.columns and len(df['II']) >= 10:\n        return int(len(df['II']) / 10)\n    return 500\n\ndef crop_to_expected_length(sig: np.ndarray, fs: int, lead: str, mode: str = 'first') -> np.ndarray:\n    target_len = int(round(fs * (10.0 if lead == 'II' else 2.5)))\n    if len(sig) == target_len:\n        return sig\n    if len(sig) < target_len:\n        # pad short signals (rare) to keep shapes consistent\n        pad = target_len - len(sig)\n        return np.pad(sig, (0, pad), mode='edge')\n    # len(sig) > target_len: crop\n    if mode == 'center':\n        start = (len(sig) - target_len) // 2\n        return sig[start:start + target_len]\n    # default: take first window\n    return sig[:target_len]\n\n# ...existing code...\ndef build_training_rows(train_df, train_dir, lead_templates, global_stats,\n                        subsample_every=1, crop_mode='first',\n                        use_windows=True, step_sec_ii=2.0, step_sec_other=0.5,\n                        min_len=5):\n    rows, y_all = [], []\n    for _, meta in train_df.iterrows():\n        csv_path = os.path.join(train_dir, str(meta['id']), f\"{meta['id']}.csv\")\n        if not os.path.exists(csv_path):\n            continue\n        try:\n            df = pd.read_csv(csv_path)\n        except Exception:\n            continue\n        fs = int(FS_MAP.get(str(meta['id']), infer_fs_from_df(df)))\n        for lead in [c for c in df.columns if c in (set(global_stats.keys()) | set(lead_templates.keys()))]:\n            sig = df[lead].dropna().values.astype(np.float32)\n\n            # Produce training segments\n            segments = (iter_windows(sig, fs, lead, crop_mode, step_sec_ii, step_sec_other)\n                        if use_windows else [crop_to_expected_length(sig, fs, lead, mode=crop_mode)])\n\n            for seg in segments:\n                n_rows = len(seg)\n                if n_rows < min_len:\n                    continue\n                feat = make_series_features(n_rows, fs, lead, lead_templates, global_stats)\n                take = np.arange(0, n_rows, subsample_every, dtype=int)\n                rows.append(feat.iloc[take].reset_index(drop=True))\n                y_all.append(pd.Series(seg[take], name='y').reset_index(drop=True))\n\n    if not rows:\n        return pd.DataFrame(), pd.Series(dtype=np.float32)\n    X = pd.concat(rows, axis=0, ignore_index=True)\n    y = pd.concat(y_all, axis=0, ignore_index=True).astype(np.float32)\n    return X, y\n# ...existing code...","metadata":{},"outputs":[],"execution_count":null},{"id":"e8384e2f","cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import butter, filtfilt\n\n# --- Utilities from your notebook idea ---\n\ndef build_lead_templates(train_df, train_dir, leads, template_len=500):\n    templates = {}\n    for lead in leads:\n        signals = []\n        for _, row in train_df.iterrows():\n            csv_path = os.path.join(train_dir, str(row['id']), f\"{row['id']}.csv\")\n            if not os.path.exists(csv_path):\n                continue\n            try:\n                df = pd.read_csv(csv_path)\n                if lead not in df.columns:\n                    continue\n                s = df[lead].dropna().values.astype(np.float32)\n                if len(s) < 50:\n                    continue\n                s_norm = (s - s.mean()) / (s.std() + 1e-8)\n                s_resamp = np.interp(\n                    np.linspace(0, 1, template_len),\n                    np.linspace(0, 1, len(s_norm)),\n                    s_norm\n                )\n                signals.append(s_resamp)\n            except:\n                continue\n        if signals:\n            templates[lead] = np.mean(signals, axis=0)\n        else:\n            t = np.linspace(0, 1, template_len)\n            templates[lead] = np.sin(2 * np.pi * t)\n    return templates\n","metadata":{},"outputs":[],"execution_count":null},{"id":"3218e18e","cell_type":"code","source":"def compute_global_stats(train_df, train_dir):\n    all_stats = {}\n    for _, row in train_df.iterrows():\n        csv_path = os.path.join(train_dir, str(row['id']), f\"{row['id']}.csv\")\n        if not os.path.exists(csv_path):\n            continue\n        try:\n            df = pd.read_csv(csv_path)\n            for lead in df.columns:\n                vals = df[lead].dropna().values\n                if len(vals) == 0:\n                    continue\n                if lead not in all_stats:\n                    all_stats[lead] = []\n                all_stats[lead].extend(vals.tolist())\n        except:\n            continue\n    global_stats = {}\n    for lead, vals in all_stats.items():\n        v = np.asarray(vals, dtype=np.float32)\n        if len(v) == 0:\n            continue\n        global_stats[lead] = dict(\n            mean=float(np.mean(v)),\n            std=float(np.std(v) if len(v) > 1 else 0.1),\n            median=float(np.median(v)),\n            min=float(np.min(v)),\n            max=float(np.max(v)),\n        )\n    return global_stats\n\ndef lowpass_15hz(x, fs):\n    if len(x) < 10:\n        return x\n    nyq = 0.5 * fs\n    wn = min(15.0 / nyq, 0.99)\n    b, a = butter(2, wn, btype='low')\n    return filtfilt(b, a, x)\n\n# --- Feature extraction per (id, lead) series ---\n\ndef rolling_mean(x, w):\n    if w <= 1:\n        return x\n    k = np.ones(w, dtype=np.float32) / w\n    return np.convolve(x, k, mode='same')\n\ndef make_series_features(n_rows, fs, lead, lead_templates, global_stats, template_len=500):\n    # Meta\n    duration = 10.0 if lead == 'II' else 2.5\n    idx = np.arange(n_rows, dtype=np.int32)\n    i_norm = idx / max(n_rows - 1, 1)\n    t_sec = i_norm * duration\n\n    # Sin/Cos bases\n    freqs = [0.5, 1, 2, 3, 5, 7, 10]\n    max_f = fs / 2.0\n    sin_feats, cos_feats = [], []\n    for f in freqs:\n        if f < max_f:\n            sin_feats.append(np.sin(2 * np.pi * f * t_sec))\n            cos_feats.append(np.cos(2 * np.pi * f * t_sec))\n    if len(sin_feats) == 0:\n        sin_feats = [np.zeros(n_rows)]\n        cos_feats = [np.zeros(n_rows)]\n    sin_feats = np.vstack(sin_feats).T\n    cos_feats = np.vstack(cos_feats).T\n\n    # Beat phase windows\n    heart_period = 0.8\n    phase = np.mod(t_sec, heart_period) / heart_period\n    p_mask   = ((phase >= 0.00) & (phase <= 0.12)).astype(np.float32)\n    qrs_mask = ((phase >= 0.20) & (phase <= 0.28)).astype(np.float32)\n    t_mask   = ((phase >= 0.40) & (phase <= 0.60)).astype(np.float32)\n\n    # Template and derivatives\n    tpl_base = lead_templates.get(lead, next(iter(lead_templates.values())))\n    tpl = np.interp(np.linspace(0, 1, n_rows), np.linspace(0, 1, len(tpl_base)), tpl_base)\n    tpl_d1 = np.gradient(tpl)\n    tpl_d2 = np.gradient(tpl_d1)\n\n    # Local windows\n    m5  = rolling_mean(tpl, 5)\n    m11 = rolling_mean(tpl, 11)\n    m31 = rolling_mean(tpl, 31)\n    dev5  = tpl - m5\n    dev11 = tpl - m11\n    dev31 = tpl - m31\n\n    # Band-limited version\n    tpl_lp = lowpass_15hz(tpl, fs)\n    tpl_lp_d1 = np.abs(np.gradient(tpl_lp))\n\n    # Lead stats (broadcast)\n    stats = global_stats.get(lead, dict(mean=0.0, std=0.1, median=0.0, min=-0.1, max=0.1))\n    amp_range = stats['max'] - stats['min']\n    coeff_var = stats['std'] / (abs(stats['mean']) + 1e-8)\n\n    # Assemble features\n    feat = {\n        'i': idx,\n        'i_norm': i_norm,\n        't_sec': t_sec,\n        't_sec2': t_sec**2,\n        't_sec3': t_sec**3,\n        'is_lead_II': (lead == 'II') * 1.0,\n        'fs': np.full(n_rows, fs, dtype=np.float32),\n        'n_rows': np.full(n_rows, n_rows, dtype=np.int32),\n        'duration': np.full(n_rows, duration, dtype=np.float32),\n        'phase': phase,\n        'p_mask': p_mask,\n        'qrs_mask': qrs_mask,\n        't_mask': t_mask,\n        'tpl': tpl,\n        'tpl_d1': tpl_d1,\n        'tpl_d2': tpl_d2,\n        'tpl_m5': m5,\n        'tpl_m11': m11,\n        'tpl_m31': m31,\n        'tpl_dev5': dev5,\n        'tpl_dev11': dev11,\n        'tpl_dev31': dev31,\n        'tpl_lp': tpl_lp,\n        'tpl_lp_d1_abs': tpl_lp_d1,\n        'lead_mean': np.full(n_rows, stats['mean'], dtype=np.float32),\n        'lead_std': np.full(n_rows, stats['std'], dtype=np.float32),\n        'lead_median': np.full(n_rows, stats['median'], dtype=np.float32),\n        'lead_min': np.full(n_rows, stats['min'], dtype=np.float32),\n        'lead_max': np.full(n_rows, stats['max'], dtype=np.float32),\n        'lead_amp_range': np.full(n_rows, amp_range, dtype=np.float32),\n        'lead_coeff_var': np.full(n_rows, coeff_var, dtype=np.float32),\n        'dist_start': idx,\n        'dist_end': (n_rows - 1 - idx),\n        'dist_start_norm': idx / max(n_rows - 1, 1),\n        'dist_end_norm': (n_rows - 1 - idx) / max(n_rows - 1, 1),\n        'lead_cat': np.full(n_rows, lead, dtype=object),  # treat as categorical in CatBoost/LGBM\n    }\n\n    # Add sin/cos bases with names\n    for k in range(sin_feats.shape[1]):\n        feat[f'sin_{k}'] = sin_feats[:, k]\n        feat[f'cos_{k}'] = cos_feats[:, k]\n\n    return pd.DataFrame(feat)\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"3a1815c7","cell_type":"markdown","source":"# Train a tree model (CatBoost) with 80/20 split and evaluate with the competition score\n","metadata":{}},{"id":"02520daf","cell_type":"code","source":"# Group-aware 80/20 split on ECG ids\nall_ids = train['id'].unique().tolist()\nlimit_ids = all_ids[:Config.LIMIT_IDS] if Config.LIMIT_IDS else all_ids\n\ngss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=Config.SEED)\ntrain_idx, val_idx = next(gss.split(np.zeros(len(limit_ids)), groups=limit_ids))\ntrain_ids = [limit_ids[i] for i in train_idx]\nval_ids = [limit_ids[i] for i in val_idx]\nprint('Train ids:', len(train_ids), 'Val ids:', len(val_ids))\n# Build training matrices (limited subset)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"25b6f50c","cell_type":"code","source":"\nlead_templates = build_lead_templates(train[train['id'].isin(train_ids)], TRAIN_DIR, LEADS, template_len=500)\nglobal_stats   = compute_global_stats(train[train['id'].isin(train_ids)], TRAIN_DIR)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"32726e61","cell_type":"code","source":"\nX_train, y_train = build_training_rows(train[train['id'].isin(train_ids)], TRAIN_DIR, lead_templates, global_stats, subsample_every=Config.SUBSAMPLE_EVERY)\nX_val, y_val     = build_training_rows(train[train['id'].isin(val_ids)], TRAIN_DIR, lead_templates, global_stats, subsample_every=Config.SUBSAMPLE_EVERY)\nprint('X_train:', X_train.shape, 'y_train:', y_train.shape)\nprint('X_val:', X_val.shape, 'y_val:', y_val.shape)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"79bf963d","cell_type":"code","source":"# check NaN\nprint('Train NaN:', X_train.isna().sum().sum(), 'Val NaN:', X_val.isna().sum().sum())\n# check NaN in y\nprint('y_train NaN:', y_train.isna().sum(), 'y_val NaN:', y_val.isna().sum())","metadata":{},"outputs":[],"execution_count":null},{"id":"192e2ab2","cell_type":"code","source":"# One-hot encode categorical columns and align columns\ncat_cols = [c for c in X_train.columns if X_train[c].dtype == 'object']\nX_train = pd.get_dummies(X_train, columns=cat_cols, dtype=np.float32)\nX_val   = pd.get_dummies(X_val,   columns=cat_cols, dtype=np.float32)\n\n# Align columns\nDUMMY_COLUMNS = X_train.columns\nX_val = X_val.reindex(columns=DUMMY_COLUMNS, fill_value=0)\n\n# Scale features\nSCALER = StandardScaler()\nX_train = pd.DataFrame(SCALER.fit_transform(X_train), columns=DUMMY_COLUMNS).astype(np.float32)\nX_val   = pd.DataFrame(SCALER.transform(X_val),   columns=DUMMY_COLUMNS).astype(np.float32)\n\nprint('After dummies+scaling -> X_train:', X_train.shape, 'X_val:', X_val.shape)\n\n# Identify categorical feature names for reference (now encoded)\nprint('Encoded categorical columns:', [c for c in DUMMY_COLUMNS if 'lead_cat_' in c])\nX_train = X_train.reset_index(drop=True)\nX_val   = X_val.reset_index(drop=True)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"b097436f","cell_type":"code","source":"\nmodel = CatBoostRegressor(\n    loss_function='RMSE',\n    learning_rate=0.01,\n    depth=6,\n    l2_leaf_reg=3.0,\n    iterations=Config.ITERATIONS,\n    random_seed=Config.SEED,\n    od_type='Iter',\n    od_wait=50,\n    verbose=200,\n    task_type=\"GPU\" if Config.DEVICE != torch.device(\"cpu\") else \"CPU\",\n    \n)\n# model.fit(X_train, y_train,eval_set=(X_val, y_val))\nmodel.fit(X_train, y_train)\n# Validate with competition score on the 20% holdout\n\ndef infer_fs_from_df(df: pd.DataFrame) -> int:\n    if 'II' in df.columns:\n        L = len(df['II'])\n        return int(L / 10) if L >= 10 else 500\n    return 500\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"de692a49","cell_type":"code","source":"\ndef build_solution_and_submission_for_ids(id_list, model):\n    solution_rows = []\n    submission_rows = []\n    for image_id in id_list:\n        csv_path = os.path.join(TRAIN_DIR, str(image_id), f\"{image_id}.csv\")\n        if not os.path.exists(csv_path):\n            continue\n        try:\n            df_ecg = pd.read_csv(csv_path)\n        except Exception:\n            continue\n        fs = infer_fs_from_df(df_ecg)\n        for lead in LEADS:\n            if lead not in df_ecg.columns:\n                continue\n            # y_true = df_ecg[lead].values.astype(np.float32)\n            y_true = df_ecg[lead].dropna().values.astype(np.float32)  # Drop NaN values\n            n_rows = len(y_true)\n            # Check for NaN in y_true\n            # if np.any(np.isnan(y_true)):\n            #     print(f\"Warning: NaN in ground truth for {image_id} lead {lead}. Filling with 0.\")\n            #     y_true = np.nan_to_num(y_true)  # Replace NaN with 0\n            feats = make_series_features(n_rows, fs, lead, lead_templates, global_stats)\n\n            # Preprocess: one-hot encode object columns and scale using training artifacts\n            obj_cols = [c for c in feats.columns if feats[c].dtype == 'object']\n            feats_proc = pd.get_dummies(feats, columns=obj_cols, dtype=np.float32)\n            feats_proc = feats_proc.reindex(columns=DUMMY_COLUMNS, fill_value=0)\n            feats_scaled = pd.DataFrame(SCALER.transform(feats_proc), columns=DUMMY_COLUMNS).astype(np.float32)\n            #stat of feats NaN\n            na = feats_scaled.isna().sum()\n            if na.sum() > 0:\n                print(f\"Warning: NaN in features for {image_id} lead {lead}:\")\n                print(na[na > 0])\n            y_pred = model.predict(feats_scaled)\n            if len(y_pred) != n_rows:\n                print(f\"Warning: prediction length mismatch for {image_id} lead {lead}: expected {n_rows}, got {len(y_pred)}\")\n                continue\n            if np.any(np.isnan(y_pred)):\n                print(f\"Warning: NaN in predictions for {image_id} lead {lead}\")\n            for i in range(n_rows):\n                rid = f\"{image_id}_{i}_{lead}\"\n                solution_rows.append({'id': rid, 'fs': fs, 'value': float(y_true[i])})\n                submission_rows.append({'id': rid, 'value': float(y_pred[i])})\n    # fill NaN for solution\n    solution_rows = pd.DataFrame(solution_rows)\n    return pd.DataFrame(solution_rows), pd.DataFrame(submission_rows)\n\nsol_val, sub_val = build_solution_and_submission_for_ids(val_ids, model)\nprint('Validation rows:', sol_val.shape, sub_val.shape)\nval_score = score(sol_val, sub_val, row_id_column_name='id')\nprint('Validation competition score:', val_score)","metadata":{},"outputs":[],"execution_count":null},{"id":"fff0dfb8","cell_type":"markdown","source":"## make submission","metadata":{}},{"id":"fb426bde","cell_type":"code","source":"# Predict on test set and write submission.csv\nprint('Predicting on test set and writing submission.csv ...')\n\npredictions = {}\nfor _, row in test.iterrows():\n    base_id = row['id']\n    lead    = row['lead']\n    n_rows  = int(row['number_of_rows'])\n    fs      = int(row.get('fs', 500))\n\n    feats = make_series_features(n_rows, fs, lead, lead_templates, global_stats)\n\n    # Preprocess test features with full-training artifacts\n    obj_cols = [c for c in feats.columns if feats[c].dtype == 'object']\n    feats_proc = pd.get_dummies(feats, columns=obj_cols, dtype=np.float32)\n    feats_proc = feats_proc.reindex(columns=DUMMY_COLUMNS, fill_value=0)\n    feats_scaled = pd.DataFrame(SCALER.transform(feats_proc), columns=DUMMY_COLUMNS).astype(np.float32)\n\n    y_pred = model.predict(feats_scaled)\n    # Optional post-filtering for smoother morphology (15 Hz low-pass)\n    y_pred = lowpass_15hz(y_pred, fs) if len(y_pred) >= 10 else y_pred\n    predictions[(base_id, lead)] = y_pred.astype(np.float32)\n\nsubmission_data = []\nfor _, row in test.iterrows():\n    base_id = row['id']\n    lead    = row['lead']\n    n_rows  = int(row['number_of_rows'])\n    sig     = predictions[(base_id, lead)]\n    for i in range(n_rows):\n        submission_data.append({'id': f\"{base_id}_{i}_{lead}\", 'value': float(sig[i])})\n\nsubmission_df = pd.DataFrame(submission_data)\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"91673ab2","cell_type":"code","source":"submission_df.shape","metadata":{},"outputs":[],"execution_count":null}]}