{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":41880,"databundleVersionId":5677426,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MJF-1B: Parkinson's Freezing of Gait Prediction\n\nKaggle competition dataset: https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/data\n\nObjective: \nTo detect the start and stop of each freezing episode and the occurence in these series of three types of freezing of gait events:\n* Start Hesitation\n* Turn\n* Walking","metadata":{}},{"cell_type":"markdown","source":"# Milestone 1 & 2  \n\nObjectives: \n* Load and label data (attach patient_id)\n* Ensure consistent units (convert g → m/s²)\n* Apply a zero-phase band-pass filter (0.1 - 30 Hz) per patient.\n* Derive a directionless signal (Acc_MAGNITUDE)\n* Standardize channels per patient (z-score).","metadata":{}},{"cell_type":"markdown","source":"# Import Python Libraries\n* **numpy/pandas**: Numeric and tabular utilities\n* **polars**: Fast DataFrame ingestion and column ops\n* **matplotlib/seaborn**: Visualization\n* **scipy.signal**: Digital filter design and zero-phase\n* **pathlib/os**: File system handling\n* **sklearn**: StandardScaler","metadata":{}},{"cell_type":"code","source":"import numpy as np                   \nimport pandas as pd                 \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.signal import butter, filtfilt\nimport tsfresh\nimport os\nfrom sklearn.preprocessing import StandardScaler\nimport polars as pl\nimport dask.dataframe as dd\nfrom pathlib import Path            ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:43.638545Z","iopub.execute_input":"2025-09-19T19:22:43.638869Z","iopub.status.idle":"2025-09-19T19:22:52.165786Z","shell.execute_reply.started":"2025-09-19T19:22:43.638842Z","shell.execute_reply":"2025-09-19T19:22:52.164854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"# File paths for training datasets\ndefog   = Path('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/defog')\nnotype  = Path('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/notype')\ntdcsfog = Path('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/train/tdcsfog')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:52.167173Z","iopub.execute_input":"2025-09-19T19:22:52.168220Z","iopub.status.idle":"2025-09-19T19:22:52.172097Z","shell.execute_reply.started":"2025-09-19T19:22:52.168192Z","shell.execute_reply":"2025-09-19T19:22:52.171268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List all files, folders, and subfolders\nall_files = os.listdir('../input/tlvmc-parkinsons-freezing-gait-prediction')\nprint('All competition datasets:')\nprint(all_files)\n\nprint()\n\ntrain_files = os.listdir('../input/tlvmc-parkinsons-freezing-gait-prediction/train')\nprint('Folders in train:')\nprint(train_files)\n\nprint() \n\ndefog_files = os.listdir(defog)\nprint('First 10 files in defog:')\nprint(defog_files[:10])\nprint(f'Files remaining: {len(defog_files)-10}')\n\nprint() \n\ntdcsfog_files = os.listdir(tdcsfog)\nprint('First 10 files in tdcsfog:')\nprint(tdcsfog_files[:10])\nprint(f'Files remaining: {len(tdcsfog_files)-10}')\n\nprint() \n\nnotype_files = os.listdir(notype)\nprint('First 10 files in notype:')\nprint(notype_files[:10])\nprint(f'Files remaining: {len(notype_files)-10}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:52.172835Z","iopub.execute_input":"2025-09-19T19:22:52.173054Z","iopub.status.idle":"2025-09-19T19:22:52.236488Z","shell.execute_reply.started":"2025-09-19T19:22:52.173036Z","shell.execute_reply":"2025-09-19T19:22:52.235699Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# General Helper Functions ","metadata":{}},{"cell_type":"code","source":"def load_files(folder_path: Path) -> pl.DataFrame:\n    \"\"\"\n    Loads all CSVs from a folder into a single Polars DataFrame.\n    Adds patient_id from file filename. \n    \"\"\"\n    \n    df_list = []\n    for path in folder_path.glob(\"*.csv\"): \n        patient_id = path.stem\n        df = pl.read_csv(path)\n        df = df.with_columns(pl.lit(patient_id).alias(\"patient_id\"))\n        df_list.append(df)\n    return pl.concat(df_list) if df_list else pl.DataFrame()\n    \n\ndef convert_valid_and_task(df: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"\n    Cast Valid and Task columns to Int8 if present.\n    \"\"\"\n    out = df\n    if \"Valid\" in out.columns:\n        out = out.with_columns(pl.col(\"Valid\").cast(pl.Int8).alias(\"Valid\"))\n    if \"Task\" in out.columns:\n        out = out.with_columns(pl.col(\"Task\").cast(pl.Int8).alias(\"Task\"))\n    return out\n    \n\ndef add_acc_magnitude(df: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"\n    Acc magnitude = sqrt(AccV^2 + AccML^2 + AccAP^2).\n    \"\"\"\n    \n    return df.with_columns(\n        ((pl.col(\"AccV\")**2 + pl.col(\"AccML\")**2 + pl.col(\"AccAP\")**2).sqrt())\n        .alias(\"Acc_MAGNITUDE\")\n    )\n\n\ndef standardize_acc_by_patient(df: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"\n    Z-score AccV/AccML/AccAP per patient_id.\n    \"\"\"\n    \n    acc_cols = [\"AccV\", \"AccML\", \"AccAP\"]\n    out = df\n    for col in acc_cols:\n        if col in out.columns:\n            out = out.with_columns(\n                ((pl.col(col) - pl.col(col).mean().over(\"patient_id\")) /\n                 pl.col(col).std().over(\"patient_id\")).alias(col)\n            )\n    return out\n\ndef detect_outliers(df: pl.DataFrame, z_thresh: float = 3.0) -> pl.DataFrame:\n    \"\"\"\n    Return rows where any accel channel has |z| > z_thresh (after standardization).\n    \"\"\"\n    \n    acc_cols = [\"AccV\", \"AccML\", \"AccAP\"]\n    # Build a boolean mask across columns that exist\n    masks = []\n    for c in acc_cols:\n        if c in df.columns:\n            masks.append(pl.col(c).abs() > z_thresh)\n    return df.filter(pl.any_horizontal(masks)) if masks else df.head(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:52.238090Z","iopub.execute_input":"2025-09-19T19:22:52.238410Z","iopub.status.idle":"2025-09-19T19:22:52.249927Z","shell.execute_reply.started":"2025-09-19T19:22:52.238389Z","shell.execute_reply":"2025-09-19T19:22:52.248945Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Filtering Helper Functions \n* Low-cut (0.1 Hz) removes drift/sensor bias (very slow trends)\n* High-cut (30 Hz) removes strap/device noise (sudden spikes)\n* filtfilt: (forward+backward) yields zero phase → no event time shift.\n\nWe infer sampling frequency from the *Time* column to adapt to slight rate differences ","metadata":{}},{"cell_type":"code","source":"def infer_fs(time_series: np.ndarray) -> float:\n    \"\"\"\n    Infer sampling frequency (Hz) from a time array (seconds) using median Δt.\n    \"\"\"\n    \n    # Avoid zeros / NaNs\n    dt = np.diff(time_series.astype(float))\n    dt = dt[np.isfinite(dt) & (dt > 0)]\n    if dt.size == 0:\n        raise ValueError(\"Cannot infer sampling frequency from Time column.\")\n    return 1.0 / np.median(dt)\n\n\ndef butter_bandpass(low_hz: float, high_hz: float, fs: float, order: int = 4):\n    \"\"\"\n    Design a Butterworth band-pass filter.\n    \"\"\"\n    \n    nyq = fs / 2.0\n    low = max(1e-6, low_hz / nyq)\n    high = min(0.999999, high_hz / nyq)\n    if high <= low:\n        raise ValueError(f\"Invalid band: low={low_hz}Hz, high={high_hz}Hz for fs={fs}Hz\")\n    b, a = butter(order, [low, high], btype=\"band\")\n    return b, a\n\n\n\ndef apply_bandpass_to_polars(df: pl.DataFrame, low_hz: float = 0.1, high_hz: float = 30.0, order: int = 4) -> pl.DataFrame:\n    \"\"\"\n    Apply band-pass filter to AccV/AccML/AccAP per patient using scipy.signal.filtfilt.\n    We convert each patient's slice to pandas to interpolate small gaps, filter, then write back.\n    Sampling frequency is inferred from Time (seconds).\n    \"\"\"\n    needed = [\"Time\", \"patient_id\", \"AccV\", \"AccML\", \"AccAP\"]\n    for col in [\"Time\", \"patient_id\"]:\n        if col not in df.columns:\n            raise ValueError(f\"'{col}' must be present to filter.\")\n    # Work columns that actually exist\n    acc_cols = [c for c in [\"AccV\", \"AccML\", \"AccAP\"] if c in df.columns]\n    if not acc_cols:\n        return df\n\n    # Process per patient to keep borders clean for filtfilt\n    out_slices = []\n    for pid, pdf in df.select(needed).to_pandas().groupby(\"patient_id\", sort=False):\n        # Ensure sorted by time\n        pdf = pdf.sort_values(\"Time\", kind=\"mergesort\")\n        fs = infer_fs(pdf[\"Time\"].to_numpy())\n        b, a = butter_bandpass(low_hz, high_hz, fs, order)\n\n        # Interpolate small gaps and filter each channel\n        for col in acc_cols:\n            # to_numeric + interpolate to avoid NaNs breaking filtfilt\n            sig = pd.to_numeric(pdf[col], errors=\"coerce\").interpolate(limit_direction=\"both\").to_numpy(dtype=float)\n            # If the segment is too short, skip filtering\n            if sig.size > max(3 * max(len(b), len(a)), 20):\n                sig_f = filtfilt(b, a, sig, method=\"pad\")\n            else:\n                sig_f = sig\n            pdf[col] = sig_f\n\n        out_slices.append(pdf)\n\n    # Merge all filtered patient slices back\n    filtered_pd = pd.concat(out_slices, ignore_index=True)\n\n    # Join the filtered columns back onto original Polars df (keeping any extra columns intact)\n    filtered_pl = pl.from_pandas(filtered_pd[[\"Time\", \"patient_id\"] + acc_cols])\n    out = df.join(filtered_pl, on=[\"Time\", \"patient_id\"], how=\"left\", suffix=\"_f\")\n    # Overwrite originals with filtered where available\n    for col in acc_cols:\n        fcol = f\"{col}_f\"\n        out = out.with_columns(\n            pl.when(pl.col(fcol).is_not_null()).then(pl.col(fcol)).otherwise(pl.col(col)).alias(col)\n        ).drop(fcol)\n    return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:52.250917Z","iopub.execute_input":"2025-09-19T19:22:52.251239Z","iopub.status.idle":"2025-09-19T19:22:52.273956Z","shell.execute_reply.started":"2025-09-19T19:22:52.251210Z","shell.execute_reply":"2025-09-19T19:22:52.273092Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training dataset ","metadata":{}},{"cell_type":"code","source":"defog_df   = load_files(defog)\nnotype_df  = load_files(notype)\ntdcsfog_df = load_files(tdcsfog)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:22:52.275062Z","iopub.execute_input":"2025-09-19T19:22:52.275388Z","iopub.status.idle":"2025-09-19T19:23:39.141822Z","shell.execute_reply.started":"2025-09-19T19:22:52.275357Z","shell.execute_reply":"2025-09-19T19:23:39.140815Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Review Data ","metadata":{}},{"cell_type":"code","source":"print(defog_df.head())\nprint(defog_df.describe())\nprint(defog_df.shape)\nprint(defog_df.columns)\nprint(defog_df.dtypes)\n\nprint(notype_df.head())\nprint(notype_df.shape)\nprint(notype_df.columns)\nprint(notype_df.dtypes)\nprint(notype_df.describe())\n\nprint(tdcsfog_df.head())\nprint(tdcsfog_df.shape)\nprint(tdcsfog_df.columns)\nprint(tdcsfog_df.dtypes)\nprint(tdcsfog_df.describe())\n\nevents_df = pd.read_csv('/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction/events.csv')\nprint(events_df.head())\nprint(events_df.shape)\nprint(events_df.columns)\nprint(events_df.dtypes)\nprint(events_df.describe())\n\nunique_defog_patients = defog_df[\"patient_id\"].unique()\nprint(unique_defog_patients)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:23:39.143334Z","iopub.execute_input":"2025-09-19T19:23:39.143661Z","iopub.status.idle":"2025-09-19T19:23:43.115730Z","shell.execute_reply.started":"2025-09-19T19:23:39.143630Z","shell.execute_reply":"2025-09-19T19:23:43.114322Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Unit Conversions \n* (g -> m/s^2) for ALL training datasets\n* Models expect consistent physical units","metadata":{}},{"cell_type":"code","source":"G_CONVERSION = 9.80665\nfor df_name in [\"defog_df\", \"notype_df\", \"tdcsfog_df\"]:\n    df_tmp = locals()[df_name]\n    acc_cols = [c for c in [\"AccV\", \"AccML\", \"AccAP\"] if c in df_tmp.columns]\n    if acc_cols:\n        # Multiply in-place using with_columns to avoid SettingWithCopy issues\n        locals()[df_name] = df_tmp.with_columns([ (pl.col(c) * G_CONVERSION).alias(c) for c in acc_cols ])\n\nprint(defog_df)\nprint(notype_df)\nprint(tdcsfog_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:23:43.116846Z","iopub.execute_input":"2025-09-19T19:23:43.117170Z","iopub.status.idle":"2025-09-19T19:23:44.115250Z","shell.execute_reply.started":"2025-09-19T19:23:43.117137Z","shell.execute_reply":"2025-09-19T19:23:44.114345Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cast Valid/Task to integers where present","metadata":{}},{"cell_type":"code","source":"defog_df   = convert_valid_and_task(defog_df)\nnotype_df  = convert_valid_and_task(notype_df)\ntdcsfog_df = convert_valid_and_task(tdcsfog_df)\nprint(defog_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:23:44.116348Z","iopub.execute_input":"2025-09-19T19:23:44.116709Z","iopub.status.idle":"2025-09-19T19:23:44.342155Z","shell.execute_reply.started":"2025-09-19T19:23:44.116681Z","shell.execute_reply":"2025-09-19T19:23:44.341354Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Band-pass filter per patient \n* Removes slow drift (<0.1 Hz) and high-frequency noise (>30 Hz) while preserving gait content.\n* Uses zero-phase filtering to keep temporal alignmnet with labels/events. ","metadata":{}},{"cell_type":"code","source":"defog_df   = apply_bandpass_to_polars(defog_df,   low_hz=0.1, high_hz=30.0, order=4)\nnotype_df  = apply_bandpass_to_polars(notype_df,  low_hz=0.1, high_hz=30.0, order=4)\ntdcsfog_df = apply_bandpass_to_polars(tdcsfog_df, low_hz=0.1, high_hz=30.0, order=4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:23:44.344753Z","iopub.execute_input":"2025-09-19T19:23:44.345004Z","iopub.status.idle":"2025-09-19T19:24:32.464869Z","shell.execute_reply.started":"2025-09-19T19:23:44.344984Z","shell.execute_reply":"2025-09-19T19:24:32.463534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Acceleration magnitude \n* Vector magnitude summarizes the intensity in one channel ","metadata":{}},{"cell_type":"code","source":"defog_df   = add_acc_magnitude(defog_df)\nnotype_df  = add_acc_magnitude(notype_df)\ntdcsfog_df = add_acc_magnitude(tdcsfog_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:24:32.467160Z","iopub.execute_input":"2025-09-19T19:24:32.467735Z","iopub.status.idle":"2025-09-19T19:24:33.753570Z","shell.execute_reply.started":"2025-09-19T19:24:32.467690Z","shell.execute_reply":"2025-09-19T19:24:33.752674Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Outlier Flags \n* Quick QA to inspect extreme values that might indicate hardware glitches. ","metadata":{}},{"cell_type":"code","source":"def outlier_acc_magnitude(df: pl.DataFrame) -> pl.DataFrame:\n    df_outlier = df.with_columns((pl.col('Acc_MAGNITUDE') > 30).alias('is_outlier_mag'))\n    return df_outlier.filter(pl.col(\"is_outlier_mag\") == True)\n\nprint(outlier_acc_magnitude(defog_df))\nprint(outlier_acc_magnitude(notype_df))\nprint(outlier_acc_magnitude(tdcsfog_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:24:33.754493Z","iopub.execute_input":"2025-09-19T19:24:33.754826Z","iopub.status.idle":"2025-09-19T19:24:34.725779Z","shell.execute_reply.started":"2025-09-19T19:24:33.754797Z","shell.execute_reply":"2025-09-19T19:24:34.724801Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Standardize acceleration channels per patient (z-score)\n* Filtering first removes drift/spikes that would distord means/standard deviations.\n* Then z-scoring equalizes patients for fair modeling.  ","metadata":{}},{"cell_type":"code","source":"defog_df   = standardize_acc_by_patient(defog_df)\nnotype_df  = standardize_acc_by_patient(notype_df)\ntdcsfog_df = standardize_acc_by_patient(tdcsfog_df)\n\n# 10b) Check z-score outliers (|z| > 3)\nprint(detect_outliers(defog_df))\nprint(detect_outliers(notype_df))\nprint(detect_outliers(tdcsfog_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:24:34.726687Z","iopub.execute_input":"2025-09-19T19:24:34.726934Z","iopub.status.idle":"2025-09-19T19:24:41.043863Z","shell.execute_reply.started":"2025-09-19T19:24:34.726913Z","shell.execute_reply":"2025-09-19T19:24:41.042920Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Plot for patient #1:","metadata":{}},{"cell_type":"markdown","source":"# Plot for patient #2:","metadata":{}},{"cell_type":"markdown","source":"# Plot for patient #3:","metadata":{}},{"cell_type":"markdown","source":"# Dtype summmary ","metadata":{}},{"cell_type":"code","source":"print(f'DEFOG DATA TYPES:\\n{defog_df.dtypes}\\n')\nprint(f'TDCSFOG DATA TYPES:\\n{tdcsfog_df.dtypes}\\n')\nprint(f'NOTYPE DATA TYPES:\\n{notype_df.dtypes}\\n')\n\n# Null counts\nprint(defog_df.null_count())\nprint(notype_df.null_count())\nprint(tdcsfog_df.null_count())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-19T19:24:44.122797Z","iopub.execute_input":"2025-09-19T19:24:44.123119Z","iopub.status.idle":"2025-09-19T19:24:44.129804Z","shell.execute_reply.started":"2025-09-19T19:24:44.123085Z","shell.execute_reply":"2025-09-19T19:24:44.128671Z"}},"outputs":[],"execution_count":null}]}