{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":97984,"databundleVersionId":14096757,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 1: Basic EDA\nimport os, glob, random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\n# 1) locate the competition root under /kaggle/input\nbase_paths = [p for p in glob.glob('/kaggle/input/*') if os.path.isdir(p)]\nprint(\"Possible input roots found:\", base_paths)\n\n# Try to find train.csv and test.csv automatically\ntrain_csv_paths = glob.glob('/kaggle/input/**/train.csv', recursive=True)\ntest_csv_paths = glob.glob('/kag.glob.glob/**/test.csv', recursive=True) if False else glob.glob('/kaggle/input/**/test.csv', recursive=True)\n\nif len(train_csv_paths)==0 or len(test_csv_paths)==0:\n    print(\"Did not find train.csv or test.csv automatically. Listing top-level '/kaggle/input' for you again:\")\n    for dirname, _, filenames in os.walk('/kaggle/input'):\n        for filename in filenames[:5]:\n            print(os.path.join(dirname, filename))\nelse:\n    train_csv = train_csv_paths[0]\n    test_csv = test_csv_paths[0]\n    print(\"train.csv found at:\", train_csv)\n    print(\"test.csv found at:\", test_csv)\n\n    # 2) read metadata\n    train_df = pd.read_csv(train_csv)\n    test_df = pd.read_csv(test_csv)\n    print(\"\\n---- train.csv head ----\")\n    print(train_df.head())\n    print(\"\\ntrain.csv shape:\", train_df.shape)\n    print(\"Unique sampling frequencies in train:\", sorted(train_df['fs'].unique()))\n    print(\"sig_len stats (min,max,mean):\", train_df['sig_len'].min(), train_df['sig_len'].max(), train_df['sig_len'].mean())\n\n    print(\"\\n---- test.csv head ----\")\n    print(test_df.head())\n    print(\"\\ntest.csv shape:\", test_df.shape)\n    if 'number_of_rows' in test_df.columns:\n        print(\"Unique number_of_rows:\", sorted(test_df['number_of_rows'].unique()) )\n\n    # 3) pick a sample id from train to inspect files\n    sample_id = random.choice(train_df['id'].tolist())\n    print(\"\\nSample train id chosen:\", sample_id)\n\n    # try to find files under any folder containing this id\n    candidate_pngs = glob.glob(f\"/kaggle/input/**/{sample_id}*.png\", recursive=True)\n    candidate_csvs = glob.glob(f\"/kaggle/input/**/{sample_id}*.csv\", recursive=True)\n    print(f\"Found {len(candidate_pngs)} png(s) and {len(candidate_csvs)} csv(s) for id {sample_id}\")\n\n    if len(candidate_pngs)>0:\n        print(\"First few image paths:\")\n        for p in candidate_pngs[:8]:\n            print(\"  \", p)\n    if len(candidate_csvs)>0:\n        print(\"Signal CSV paths (first 5):\")\n        for p in candidate_csvs[:5]:\n            print(\"  \", p)\n\n    # 4) load the ground-truth CSV for that id (if present) and print info\n    gt_csv_path = None\n    for p in candidate_csvs:\n        if p.endswith(f\"{sample_id}.csv\"):\n            gt_csv_path = p\n            break\n    if gt_csv_path is None and len(candidate_csvs)>0:\n        gt_csv_path = candidate_csvs[0]\n\n    if gt_csv_path:\n        print(\"\\nLoading ground-truth CSV:\", gt_csv_path)\n        sig = pd.read_csv(gt_csv_path)\n        print(\"Signal columns:\", sig.columns.tolist())\n        print(\"Signal shape (rows x leads):\", sig.shape)\n        print(\"First 5 rows of lead II:\")\n        print(sig['II'].head().to_list()[:10])\n        # Plot lead II\n        plt.figure(figsize=(12,3))\n        plt.plot(sig['II'].values[:2000])   # show first chunk\n        plt.title(f\"Lead II (first 2000 samples) from {sample_id}\")\n        plt.xlabel(\"sample index\"); plt.ylabel(\"mV\")\n        plt.show()\n    else:\n        print(\"No ground-truth CSV found for this id in the input folders.\")\n\n    # 5) Display one PNG image (if available)\n    if len(candidate_pngs)>0:\n        img_path = candidate_pngs[0]\n        print(\"\\nDisplaying image:\", img_path)\n        img = Image.open(img_path).convert('RGB')\n        plt.figure(figsize=(10,6))\n        plt.imshow(img)\n        plt.axis('off')\n        plt.show()\n    else:\n        print(\"No PNG images found for this id.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Explore the Data\ntrain_df = pd.read_csv('/kaggle/input/physionet-ecg-image-digitization/train.csv')\ntrain_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3: Understanding The Dataset","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Explore One Example in Detail\n# import matplotlib.pyplot as plt\nimport cv2\n\n# Pick a sample id (printed one earlier)\nsample_id = 1949321289\n\n# Read a sample image\nimg_path = f'/kaggle/input/physionet-ecg-image-digitization/train/{sample_id}/{sample_id}-0004.png'\nimg = cv2.imread(img_path)\nimg_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n# Display it\nplt.imshow(img_rgb)\nplt.title(f\"ECG Image for ID: {sample_id}\")\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 5: Load and Visualize the True ECG Signal (from CSV)\n# Load the ground truth ECG data (time series)\ncsv_path = f'/kaggle/input/physionet-ecg-image-digitization/train/{sample_id}/{sample_id}.csv'\nsignal_df = pd.read_csv(csv_path)\n\n# Show the first few rows\nprint(signal_df.head())\n\n# Plot one of the leads, e.g., 'II'\nplt.figure(figsize=(12, 4))\nplt.plot(signal_df['II'][:1000])  # plot first 1000 points\nplt.title('Lead II - ECG Signal')\nplt.xlabel('Sample index')\nplt.ylabel('Voltage (mV)')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6: Understand the Task Before Modeling\n# Step 6.1: Preprocess ECG images\n# import cv2\n# import matplotlib.pyplot as plt\n\n# Pick one ECG image\nimg_path = f'/kaggle/input/physionet-ecg-image-digitization/train/{sample_id}/{sample_id}-0004.png'\n\n# Read the image\nimg = cv2.imread(img_path)\n\n# Convert to grayscale\ngray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n\n# Show both color and grayscale\nplt.figure(figsize=(12,5))\nplt.subplot(1,2,1)\nplt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\nplt.title(\"Original ECG Image\")\n\nplt.subplot(1,2,2)\nplt.imshow(gray, cmap='gray')\nplt.title(\"Grayscale ECG Image\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.1.1: Apply Thresholding or Edge Detection\n# import cv2\n# import matplotlib.pyplot as plt\n# import numpy as np\n\n# Apply adaptive thresholding\nthresh = cv2.adaptiveThreshold(\n    gray, \n    255, \n    cv2.ADAPTIVE_THRESH_GAUSSIAN_C, \n    cv2.THRESH_BINARY_INV, \n    15, \n    8\n)\n\n# Optional: clean small noise using morphological operations\nkernel = np.ones((2, 2), np.uint8)\nclean = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)\n\n# Show the results\nplt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nplt.imshow(thresh, cmap='gray')\nplt.title(\"Thresholded ECG (Inverted Binary)\")\n\nplt.subplot(1,2,2)\nplt.imshow(clean, cmap='gray')\nplt.title(\"After Morphological Cleaning\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.2 — Pixel-to-Signal Mapping\n# import numpy as np\n# import cv2\n# import matplotlib.pyplot as plt\n\n# Convert cleaned image to binary (ensure it's 0 and 1)\nbinary = (clean < 128).astype(np.uint8)\n\n# For each x-column, find the y-position of the ECG line (mean or median of white pixels)\ny_positions = []\nfor x in range(binary.shape[1]):\n    ys = np.where(binary[:, x] == 1)[0]\n    if len(ys) > 0:\n        y_positions.append(np.median(ys))\n    else:\n        y_positions.append(np.nan)\n\ny_positions = np.array(y_positions)\n\n# Normalize and invert (since higher y = lower amplitude)\nsignal_from_image = -(y_positions - np.nanmean(y_positions))\nsignal_from_image = signal_from_image / np.nanmax(np.abs(signal_from_image))\n\n# Plot the extracted waveform\nplt.figure(figsize=(12, 4))\nplt.plot(signal_from_image)\nplt.title(\"Extracted ECG Signal from Image\")\nplt.xlabel(\"Sample Index (pixels)\")\nplt.ylabel(\"Normalized Voltage\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.3 — Pairing the Extracted Signal with the True Numeric Signal\n# import pandas as pd\n# import matplotlib.pyplot as plt\n# import numpy as np\n\n# Load the true ECG signal from the CSV for the same sample_id\ncsv_path = f'/kaggle/input/physionet-ecg-image-digitization/train/{sample_id}/{sample_id}.csv'\ntrue_signal = pd.read_csv(csv_path)\n\n# Select one lead, e.g., Lead II (or whichever column has values)\nlead_name = 'II'\ntrue_signal_values = true_signal[lead_name].dropna().values\n\n# Normalize both signals for comparison\ntrue_signal_values = true_signal_values / np.max(np.abs(true_signal_values))\nimage_signal_resized = np.interp(\n    np.linspace(0, len(signal_from_image)-1, len(true_signal_values)),\n    np.arange(len(signal_from_image)),\n    signal_from_image\n)\n\n# Plot comparison\nplt.figure(figsize=(12, 5))\nplt.plot(true_signal_values, label=\"True Signal (from CSV)\")\nplt.plot(image_signal_resized, label=\"Extracted Signal (from Image)\", alpha=0.7)\nplt.title(\"Comparison: True vs Extracted ECG Signal\")\nplt.xlabel(\"Sample Index\")\nplt.ylabel(\"Normalized Voltage\")\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Aliases for Step 6.4\nextracted_signal = image_signal_resized\ntrue_signal = true_signal_values","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.4 – Learning Model to Refine Extracted ECG\n# import numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n\n# 1. Prepare Data\nX = extracted_signal.reshape(-1, 1)  # input: from image\ny = true_signal[:len(extracted_signal)].reshape(-1, 1)  # target: from CSV\n\n# normalize (optional)\nX = (X - np.mean(X)) / np.std(X)\ny = (y - np.mean(y)) / np.std(y)\n\n# reshape for CNN\nX = np.expand_dims(X, axis=0)  # (1, samples, 1)\ny = np.expand_dims(y, axis=0)\n\n# 2. Define Model\nmodel = models.Sequential([\n    layers.Conv1D(32, 5, activation='relu', padding='same', input_shape=(X.shape[1], 1)),\n    layers.Conv1D(64, 5, activation='relu', padding='same'),\n    layers.Conv1D(1, 5, activation='tanh', padding='same')\n])\n\nmodel.compile(optimizer='adam', loss='mse')\nmodel.summary()\n\n# 3. Train\nhistory = model.fit(X, y, epochs=200, verbose=0)\n\n# 4. Predict\npredicted = model.predict(X)[0].flatten()\n\n# 5. Plot\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(10,5))\nplt.plot(true_signal[:len(predicted)], label='True ECG', linewidth=2)\nplt.plot(predicted, label='Model Output (Refined)', alpha=0.8)\nplt.legend()\nplt.title('Step 6.4 - Model Refinement of Extracted ECG Signal')\nplt.xlabel('Sample Index')\nplt.ylabel('Normalized Voltage')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n# Ensure prediction and true labels have the same shape\ny_pred = model.predict(X)\n\n# Flatten both arrays\ny_pred = y_pred.flatten()\ny_true = y.flatten() if len(y.shape) > 1 else y\n\n# Make them the same length\nmin_len = min(len(y_true), len(y_pred))\ny_true = y_true[:min_len]\ny_pred = y_pred[:min_len]\n\n# Compute MSE\nmse = mean_squared_error(y_true, y_pred)\nprint(f\"Mean Squared Error (MSE): {mse:.6f}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.5 — Evaluate Model Performance\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n# Flatten y_true and y_pred for consistent shape\ny_true_flat = y_true.flatten()\ny_pred_flat = y_pred.flatten()\n\n# Compute Mean Squared Error (MSE)\nmse = mean_squared_error(y_true_flat, y_pred_flat)\nprint(f\"Mean Squared Error (MSE): {mse:.6f}\")\n\n# Compute Signal-to-Noise Ratio (SNR)\nsignal_power = np.mean(np.square(y_true_flat))\nnoise_power = np.mean(np.square(y_true_flat - y_pred_flat))\nsnr = 10 * np.log10(signal_power / noise_power)\nprint(f\"Signal-to-Noise Ratio (SNR): {snr:.2f} dB\")\n\n# Visual comparison\nplt.figure(figsize=(12,5))\nplt.plot(y_true_flat, label=\"True ECG\")\nplt.plot(y_pred_flat, label=\"Model Output (Refined)\", alpha=0.7)\nplt.title(\"Step 6.5 — Model Evaluation: True vs Predicted ECG Signal\")\nplt.xlabel(\"Sample Index\")\nplt.ylabel(\"Normalized Voltage\")\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 7.1 — Data preparation & alignment (per-record)\nfrom scipy.signal import butter, filtfilt, resample\n\n# --- Helpers ---\ndef butter_bandpass(lowcut, highcut, fs, order=4):\n    nyq = 0.5 * fs\n    low = lowcut / nyq\n    high = highcut / nyq\n    b, a = butter(order, [low, high], btype='band')\n    return b, a\n\ndef bandpass_filter(data, lowcut, highcut, fs, order=4):\n    b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n    return filtfilt(b, a, data)\n\ndef align_signals_with_maxlag(sig_true, sig_pred, fs, max_shift_seconds=0.2):\n    # returns: shifted_pred, lag_used (in samples). Positive lag means pred shifts right (delayed).\n    maxlag = int(round(max_shift_seconds * fs))\n    # handle NaNs in pred by zero-filling\n    sig_pred_clean = np.copy(sig_pred)\n    sig_pred_clean[np.isnan(sig_pred_clean)] = 0.0\n    # cross-correlation\n    corr = np.correlate(sig_true - np.mean(sig_true), sig_pred_clean - np.mean(sig_pred_clean), mode='full')\n    lags = np.arange(-len(sig_pred_clean)+1, len(sig_true))\n    # find best lag but restrict to [-maxlag, +maxlag]\n    center = len(corr)//2\n    window_idx = np.where((lags >= -maxlag) & (lags <= maxlag))[0]\n    if len(window_idx)==0:\n        best_idx = np.argmax(corr)\n    else:\n        best_idx = window_idx[np.argmax(corr[window_idx])]\n    best_lag = lags[best_idx]\n    # shift pred accordingly\n    if best_lag > 0:\n        shifted = np.concatenate((np.zeros(best_lag), sig_pred_clean[:-best_lag]))\n    elif best_lag < 0:\n        shifted = np.concatenate((sig_pred_clean[-best_lag:], np.zeros(-best_lag)))\n    else:\n        shifted = sig_pred_clean\n    return shifted, int(best_lag)\n\n# --- Choose a sample (use your current sample_id) ---\n# If you restarted kernel, set sample_id explicitly:\n# sample_id = 49746380\n# Otherwise the variable should already exist. I'll try to read it safely:\ntry:\n    sample_id\nexcept NameError:\n    sample_id = int(input(\"Enter sample_id to process (e.g. 49746380): \"))\n\nprint(\"Processing sample_id =\", sample_id)\n\n# paths\nroot = '/kaggle/input/physionet-ecg-image-digitization'\ntrain_csv = os.path.join(root, 'train.csv')\ntrain_df = pd.read_csv(train_csv)\nfs = int(train_df.loc[train_df['id'] == sample_id, 'fs'].values[0])\nprint(\"Sampling frequency (fs):\", fs)\n\n# load true signal\ncsv_path = glob.glob(os.path.join(root, 'train', str(sample_id), f\"{sample_id}.csv\"))\nif len(csv_path)==0:\n    raise FileNotFoundError(\"Ground-truth CSV not found for this sample_id\")\ncsv_path = csv_path[0]\ntrue_df = pd.read_csv(csv_path)\n# choose lead II (good long one)\nlead = 'II'\nif lead not in true_df.columns:\n    # fall back to first non-NaN column\n    nonnan_cols = [c for c in true_df.columns if not true_df[c].isna().all()]\n    lead = nonnan_cols[0]\nprint(\"Using lead:\", lead)\ntrue_signal = true_df[lead].dropna().values\nsig_len = len(true_signal)\nprint(\"True signal length:\", sig_len)\n\n# load and extract image-derived signal (reuse your earlier clean/extract code)\n# try a few png variants in the folder and pick one available (use -0004 if present)\nimg_candidates = glob.glob(os.path.join(root, 'train', str(sample_id), f\"{sample_id}-*.png\"))\nif len(img_candidates)==0:\n    raise FileNotFoundError(\"No PNGs found in sample folder\")\nimg_path = img_candidates[0]\nprint(\"Using image:\", img_path)\n\n# read image, grayscale, threshold & clean (same approach you used)\nimg = cv2.imread(img_path)\ngray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\nthresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 15, 8)\nkernel = np.ones((2,2), np.uint8)\nclean = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)\n\n# extract column-wise median y position (same approach)\nbinary = (clean > 0).astype(np.uint8)  # 1 where trace/grid exists\ny_positions = []\nfor x in range(binary.shape[1]):\n    ys = np.where(binary[:, x] == 1)[0]\n    if len(ys) > 0:\n        y_positions.append(np.median(ys))\n    else:\n        y_positions.append(np.nan)\ny_positions = np.array(y_positions)\n# invert and normalize\nimg_signal = -(y_positions - np.nanmean(y_positions))\n# replace NaN with linear interpolation\nnans = np.isnan(img_signal)\nif nans.any():\n    img_signal[nans] = np.interp(np.flatnonzero(nans), np.flatnonzero(~nans), img_signal[~nans])\n# normalize to [-1,1]\nimg_signal = img_signal / np.nanmax(np.abs(img_signal))\n\nprint(\"Raw image-derived signal length (pixels):\", len(img_signal))\n\n# Resample image-derived signal to match the number of samples in true_signal\nimg_len = len(img_signal)\nresampled_img_signal = np.interp(\n    np.linspace(0, img_len-1, sig_len),\n    np.arange(img_len),\n    img_signal\n)\n\n# Band-pass filter both signals (0.5 - 40 Hz) using the record's fs\nlowcut, highcut = 0.5, 40.0\ntry:\n    true_filtered = bandpass_filter(true_signal, lowcut, highcut, fs, order=4)\nexcept Exception as e:\n    # if filter fails due to short length, skip filtering\n    print(\"Warning: true signal filtering failed:\", e)\n    true_filtered = true_signal.copy()\n\n# For the image-resampled signal, we must use the same fs and length. If true length != expected time*fs,\n# assume true_signal length is correct and resampled signal already matches it.\ntry:\n    img_filtered = bandpass_filter(resampled_img_signal, lowcut, highcut, fs, order=4)\nexcept Exception as e:\n    print(\"Warning: image signal filtering failed:\", e)\n    img_filtered = resampled_img_signal.copy()\n\n# Normalize to zero mean (competition removes constant offset anyway)\ntrue_norm = (true_filtered - np.mean(true_filtered)) / (np.std(true_filtered) + 1e-12)\nimg_norm = (img_filtered - np.mean(img_filtered)) / (np.std(img_filtered) + 1e-12)\n\n# Align image->true using cross-correlation but limit lag to 0.2 seconds (competition rule)\naligned_img, lag_used = align_signals_with_maxlag(true_norm, img_norm, fs, max_shift_seconds=0.2)\nprint(\"Lag used (samples):\", lag_used, \"which is\", lag_used / fs, \"seconds\")\n\n# After alignment, re-normalize amplitude by matching std or scale\nscale = (np.std(true_norm) + 1e-12) / (np.std(aligned_img) + 1e-12)\naligned_img = aligned_img * scale\n\n# Compute a baseline SNR between true and aligned image signal\nsignal_power = np.mean(true_norm**2)\nnoise_power = np.mean((true_norm - aligned_img)**2)\nsnr_db = 10 * np.log10(signal_power / (noise_power + 1e-12))\nprint(f\"Baseline aligned SNR (after filtering & alignment): {snr_db:.3f} dB\")\n\n# Plot quick comparison\nplt.figure(figsize=(12,4))\nplt.plot(true_norm, label='True (filtered, norm)')\nplt.plot(aligned_img, label='Image-derived (filtered, aligned)', alpha=0.8)\nplt.title(f'Sample {sample_id} — True vs Image-derived (aligned). SNR={snr_db:.3f} dB')\nplt.legend()\nplt.show()\n\n# Save prepared arrays (for next steps)\nprepared = {\n    'sample_id': sample_id,\n    'fs': fs,\n    'lead': lead,\n    'true_norm': true_norm,\n    'img_aligned': aligned_img,\n    'snr_db': snr_db\n}\nprint(\"Prepared keys:\", list(prepared.keys()))\n# keep variables in notebook for next steps\ntrue_norm, aligned_img, fs, lead, sample_id, snr_db","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n#  STEP 7.2 — Feature Engineering + Segmentation + Training\n# =========================================================\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n\n# -------------------------------\n#   Helper: Add Features\n# -------------------------------\n# def add_features(ecg_signal):\n#     ecg_norm = (ecg_signal - np.mean(ecg_signal)) / np.std(ecg_signal)\n#     d_ecg = np.gradient(ecg_norm)\n#     win = 128\n#     spec_energy = np.convolve(ecg_norm**2, np.ones(win)/win, mode='same')\n#     X_feat = np.stack([ecg_norm, d_ecg, spec_energy], axis=-1)\n#     return X_feat\n\ndef add_features(ecg_signal, fs=300, include_spectral=True):\n    # import numpy as np\n    from scipy.signal import welch\n\n    # Normalize the signal\n    ecg_norm = (ecg_signal - np.mean(ecg_signal)) / (np.std(ecg_signal) + 1e-8)\n\n    # Derivative (temporal feature)\n    ecg_diff = np.diff(ecg_norm, prepend=ecg_norm[0])\n\n    # Initialize feature list with time-domain features\n    features = [ecg_norm, ecg_diff]\n\n    # Optionally add spectral features\n    if include_spectral:\n        f, Pxx = welch(ecg_norm, fs=fs, nperseg=256)\n        # Interpolate to match signal length\n        spectral_energy = np.interp(\n            np.linspace(0, len(ecg_norm), len(ecg_norm)),\n            np.linspace(0, len(Pxx), len(Pxx)),\n            Pxx\n        )\n        features.append(spectral_energy)\n\n    # Stack all features column-wise (shape: T x channels)\n    X = np.stack(features, axis=-1)\n\n    return X\n\n# -------------------------------\n#   Helper: Segment Signal\n# -------------------------------\ndef segment_signal(X, y, window_size=500, step=250):\n    X_segments, y_segments = [], []\n    for i in range(0, len(X) - window_size, step):\n        X_segments.append(X[i:i+window_size])\n        y_segments.append(y[i:i+window_size])\n    return np.array(X_segments), np.array(y_segments)\n\n# -------------------------------\n#   Build CNN + BiLSTM Model\n# -------------------------------\ndef build_model(input_shape):\n    inputs = layers.Input(shape=input_shape, name=\"ecg_input\")\n\n    x = layers.Conv1D(64, 7, padding=\"same\")(inputs)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Dropout(0.2)(x)\n\n    x = layers.Conv1D(128, 5, padding=\"same\")(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Dropout(0.2)(x)\n\n    x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x)\n    x = layers.Dropout(0.2)(x)\n    x = layers.Bidirectional(layers.LSTM(32, return_sequences=True))(x)\n    x = layers.Dropout(0.2)(x)\n\n    outputs = layers.TimeDistributed(layers.Dense(1))(x)\n    model = models.Model(inputs, outputs, name=\"CNN_BiLSTM_ECG\")\n    return model\n\n# -------------------------------\n#   Load or Simulate ECG Data\n# -------------------------------\n# Replace this with your actual data arrays\n# Example: aligned_img = predicted_signal_from_phase1, true_norm = true_groundtruth\nT = 10250\naligned_img = np.random.randn(T)\ntrue_norm = aligned_img + 0.1 * np.random.randn(T)  # Simulated clean signal\n\n# Normalize both input and target\naligned_img = (aligned_img - np.mean(aligned_img)) / np.std(aligned_img)\ntrue_norm = (true_norm - np.mean(true_norm)) / np.std(true_norm)\n\n# Add derived features\nX_feat = add_features(aligned_img)\ny = true_norm.reshape(-1, 1)\n\nprint(f\"Enhanced input shape: {X_feat.shape}\")\n\n# Visualize feature channels\nfig, ax = plt.subplots(3, 1, figsize=(10, 6))\nax[0].plot(X_feat[:,0]); ax[0].set_title(\"Channel 1: Normalized ECG\")\nax[1].plot(X_feat[:,1]); ax[1].set_title(\"Channel 2: First Derivative (ΔECG)\")\nax[2].plot(X_feat[:,2]); ax[2].set_title(\"Channel 3: Spectral Energy\")\nplt.tight_layout(); plt.show()\n\n# -------------------------------\n#   Segment for Training\n# -------------------------------\nwindow_size = 500\nstep = 250\nX_segments, y_segments = segment_signal(X_feat, y, window_size, step)\nprint(f\"Segmented X: {X_segments.shape}, y: {y_segments.shape}\")\n\n# -------------------------------\n#   Compile + Train\n# -------------------------------\ndef hybrid_loss(alpha=0.3):\n    def loss_fn(y_true, y_pred):\n        mse = tf.reduce_mean(tf.square(y_true - y_pred))\n        mae = tf.reduce_mean(tf.abs(y_true - y_pred))\n        return alpha * mse + (1 - alpha) * mae\n    return loss_fn\n\ninput_shape = X_segments.shape[1:]\nmodel = build_model(input_shape)\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n    loss=hybrid_loss(alpha=0.3),\n    metrics=[tf.keras.metrics.MeanAbsoluteError()]\n)\n\ncb = [\n    ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-5, verbose=1),\n    EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n]\n\nhistory = model.fit(\n    X_segments, y_segments,\n    validation_split=0.2,\n    epochs=80,\n    batch_size=16,\n    callbacks=cb,\n    verbose=1\n)\n\n# -------------------------------\n#   Evaluate + Visualize\n# -------------------------------\npred = model.predict(X_segments[:1])\npred_flat = pred.reshape(-1)\ntrue_flat = y_segments[0].reshape(-1)\n\nmse = np.mean((pred_flat - true_flat) ** 2)\nsnr = 10 * np.log10(np.var(true_flat) / np.var(true_flat - pred_flat))\nprint(f\"\\nMSE: {mse:.4f}, SNR: {snr:.2f} dB\")\n\nplt.figure(figsize=(10, 4))\nplt.plot(true_flat, label=\"True ECG\", alpha=0.8)\nplt.plot(pred_flat, label=\"Predicted ECG\", alpha=0.8)\nplt.title(\"True vs Predicted ECG after Refinement\")\nplt.legend(); plt.tight_layout(); plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===========================================\n# Step 7.3 — Model Architecture Upgrade (CNN + BiLSTM, Optimized for Kaggle GPU)\n# ===========================================\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, callbacks, backend as K\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt\n\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\nprint(tf.config.list_physical_devices('GPU'))\n\n# --- 1) Prepare X (features) and y (target) ---\nT = len(true_norm)\nprint(\"Timesteps T =\", T)\n\n# Create features\nX_feat = add_features(aligned_img, fs=fs)  # no include_spectral to avoid argument issue\nprint(\"X_feat shape (timesteps, channels):\", X_feat.shape)\n\nn_channels = X_feat.shape[1]\nX = np.expand_dims(X_feat, axis=0).astype(np.float32)\ny = np.expand_dims(true_norm, axis=(0,2)).astype(np.float32)\n\nprint(\"X shape:\", X.shape, \"y shape:\", y.shape)\n\n# --- 2) Define lighter hybrid model (same as before, smaller filters for GPU efficiency)\ndef hybrid_model(timesteps, channels, conv_filters=(32,64), lstm_units=32, dropout=0.2):\n    inp = layers.Input(shape=(timesteps, channels), name='ecg_input')\n    x = layers.Conv1D(conv_filters[0], kernel_size=5, padding='same', activation=None)(inp)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Dropout(dropout)(x)\n\n    x = layers.Conv1D(conv_filters[1], kernel_size=3, padding='same', activation=None)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Dropout(dropout)(x)\n\n    x = layers.Bidirectional(layers.LSTM(lstm_units, return_sequences=True))(x)\n    x = layers.Dropout(dropout)(x)\n    x = layers.Bidirectional(layers.LSTM(lstm_units//2, return_sequences=True))(x)\n    x = layers.Dropout(dropout)(x)\n\n    out = layers.TimeDistributed(layers.Dense(1, activation='linear'), name='ecg_out')(x)\n    model = models.Model(inp, out, name='CNN_BiLSTM_ECG')\n    return model\n\nmodel = hybrid_model(T, n_channels)\nmodel.summary()\n\n# --- 3) Compile\ndef hybrid_loss(alpha=0.3):\n    def loss(y_true, y_pred):\n        mse = K.mean(K.square(y_true - y_pred))\n        mae = K.mean(K.abs(y_true - y_pred))\n        return mse + alpha * mae\n    return loss\n\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),\n              loss=hybrid_loss(alpha=0.3),\n              metrics=[tf.keras.metrics.MeanAbsoluteError()])\n\n# --- 4) Callbacks\nckpt_path = '/kaggle/working/best_ecg_model.h5'\ncb = [\n    callbacks.ModelCheckpoint(ckpt_path, monitor='loss', save_best_only=True, verbose=1),\n    callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=8, min_lr=1e-7, verbose=1),\n    callbacks.EarlyStopping(monitor='loss', patience=10, restore_best_weights=True, verbose=1)\n]\n\n# --- 5) Train\nhistory = model.fit(\n    X, y,\n    epochs=10,         # Start with 10 to test stability; later increase to 80\n    batch_size=1,\n    callbacks=cb,\n    verbose=2\n)\n\n# --- 6) Evaluate\ny_pred = model.predict(X)[0].flatten()\ny_true = y[0].flatten()\n\nmse_val = mean_squared_error(y_true, y_pred)\nsnr_db = 10 * np.log10(np.mean(y_true**2) / (np.mean((y_true - y_pred)**2) + 1e-12))\nprint(f\"\\nTrained model MSE: {mse_val:.6f}\")\nprint(f\"Trained model SNR: {snr_db:.3f} dB\")\n\n# --- 7) Plot\nplt.figure(figsize=(12,4))\nplt.plot(y_true, label='True ECG', linewidth=1.2)\nplt.plot(y_pred, label='Model Output (Refined)', alpha=0.8)\nplt.title(f'CNN+BiLSTM Output vs True (MSE={mse_val:.4f}, SNR={snr_db:.2f} dB)')\nplt.xlabel('Sample Index')\nplt.ylabel('Normalized Voltage')\nplt.legend()\nplt.show()\n\nprint(\"✅ Training and plotting complete!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import tensorflow as tf\n# print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n# tf.config.list_physical_devices('GPU')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Step 7.3 (Safe Debug Version - Fixed)\n# import tensorflow as tf\n# print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n# print(tf.config.list_physical_devices('GPU'))\n\n# # Reduce sequence length for fast debug\n# max_len = 2500   # temporary shorter sequence\n# X_short = X[:, :max_len, :]\n# y_short = y[:, :max_len, :]\n\n# # Extract required shapes\n# seq_len = X_short.shape[1]\n# # n_features = X_short.shape[2]\n\n# # Build smaller model (use same argument order as your earlier code)\n# model = hybrid_model(\n#     seq_len, n_features,           # instead of input_shape\n#     conv_filters=(32, 64),         # reduced filters\n#     lstm_units=32                  # smaller LSTM\n# )\n\n# # Compile model\n# model.compile(optimizer='adam', loss='mse')\n\n# # Check summary\n# model.summary()\n\n# # Quick debug training\n# history = model.fit(\n#     X_short, y_short,\n#     epochs=3,\n#     batch_size=1,\n#     verbose=1\n# )\n\n# print(\"✅ Training started successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}