{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"is_main = __name__ == '__main__'","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:40.082006Z","iopub.execute_input":"2024-01-30T01:09:40.082388Z","iopub.status.idle":"2024-01-30T01:09:40.111843Z","shell.execute_reply.started":"2024-01-30T01:09:40.082352Z","shell.execute_reply":"2024-01-30T01:09:40.110709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:40.113811Z","iopub.execute_input":"2024-01-30T01:09:40.114551Z","iopub.status.idle":"2024-01-30T01:09:40.343661Z","shell.execute_reply.started":"2024-01-30T01:09:40.114510Z","shell.execute_reply":"2024-01-30T01:09:40.342713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"if is_main:\n    df_train = pl.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:40.345118Z","iopub.execute_input":"2024-01-30T01:09:40.345524Z","iopub.status.idle":"2024-01-30T01:09:40.595482Z","shell.execute_reply.started":"2024-01-30T01:09:40.345481Z","shell.execute_reply":"2024-01-30T01:09:40.594685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    display(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:40.597177Z","iopub.execute_input":"2024-01-30T01:09:40.597542Z","iopub.status.idle":"2024-01-30T01:09:40.628505Z","shell.execute_reply.started":"2024-01-30T01:09:40.597480Z","shell.execute_reply":"2024-01-30T01:09:40.627740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scan_eeg_subsamples(eeg_id, offset):\n    return (\n        pl.scan_parquet(\n            Path('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs')\n            / f'{eeg_id}.parquet',\n        ).slice(int(offset * 200), 50 * 200)\n    )\n\ndef read_eeg_subsamples(eeg_id, offset):\n    return scan_eeg_subsamples(eeg_id, offset).collect()","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:13.354241Z","iopub.execute_input":"2024-01-30T01:10:13.354633Z","iopub.status.idle":"2024-01-30T01:10:13.360335Z","shell.execute_reply.started":"2024-01-30T01:10:13.354602Z","shell.execute_reply":"2024-01-30T01:10:13.359228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    df_eeg_0 = read_eeg_subsamples(1628180742, 0)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:13.620456Z","iopub.execute_input":"2024-01-30T01:10:13.621072Z","iopub.status.idle":"2024-01-30T01:10:13.631471Z","shell.execute_reply.started":"2024-01-30T01:10:13.621028Z","shell.execute_reply":"2024-01-30T01:10:13.630366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    display(df_eeg_0)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:15.225490Z","iopub.execute_input":"2024-01-30T01:10:15.226650Z","iopub.status.idle":"2024-01-30T01:10:15.245019Z","shell.execute_reply.started":"2024-01-30T01:10:15.226598Z","shell.execute_reply":"2024-01-30T01:10:15.243012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Butter lowpass filter","metadata":{}},{"cell_type":"markdown","source":"Learned from <https://www.kaggle.com/code/cdeotte/wavenet-starter-lb-0-52#Butter-Low-Pass-Filter>.","metadata":{}},{"cell_type":"code","source":"import scipy.signal\n\ndef butter_lowpass_filter(data, cutoff_freq = 20, sampling_rate = 200, order = 4):\n    nyquist = 0.5 * sampling_rate\n    normal_cutoff = cutoff_freq / nyquist\n    b, a = scipy.signal.butter(order, normal_cutoff, btype = 'low', analog = False)\n    return scipy.signal.lfilter(b, a, data, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:16.471958Z","iopub.execute_input":"2024-01-30T01:10:16.473082Z","iopub.status.idle":"2024-01-30T01:10:16.478185Z","shell.execute_reply.started":"2024-01-30T01:10:16.473040Z","shell.execute_reply":"2024-01-30T01:10:16.477398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    display(butter_lowpass_filter(df_eeg_0))","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:17.191655Z","iopub.execute_input":"2024-01-30T01:10:17.192512Z","iopub.status.idle":"2024-01-30T01:10:17.205027Z","shell.execute_reply.started":"2024-01-30T01:10:17.192478Z","shell.execute_reply":"2024-01-30T01:10:17.204282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    import seaborn as sns\n    import matplotlib.pyplot as plt\n    import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:17.479456Z","iopub.execute_input":"2024-01-30T01:10:17.480337Z","iopub.status.idle":"2024-01-30T01:10:17.485202Z","shell.execute_reply.started":"2024-01-30T01:10:17.480279Z","shell.execute_reply":"2024-01-30T01:10:17.484021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(pd.DataFrame(butter_lowpass_filter(df_eeg_0), columns = df_eeg_0.columns), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:17.959956Z","iopub.execute_input":"2024-01-30T01:10:17.960883Z","iopub.status.idle":"2024-01-30T01:10:23.471814Z","shell.execute_reply.started":"2024-01-30T01:10:17.960846Z","shell.execute_reply":"2024-01-30T01:10:23.470359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.slice(6000, 2000).to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(pd.DataFrame(butter_lowpass_filter(df_eeg_0)[6000:8000], columns = df_eeg_0.columns), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:23.473686Z","iopub.execute_input":"2024-01-30T01:10:23.474022Z","iopub.status.idle":"2024-01-30T01:10:26.343635Z","shell.execute_reply.started":"2024-01-30T01:10:23.473992Z","shell.execute_reply":"2024-01-30T01:10:26.342289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Downsampling","metadata":{}},{"cell_type":"markdown","source":"From discussion <https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/468684>:\n\n> In this competition, we (most likely) are only concerned with brain waves of 20Hz and below.\n\nSo downsampling from 200Hz to 40Hz would reduce the data size without loosing much information.","metadata":{}},{"cell_type":"code","source":"import scipy.signal\n\ndef downsample(data, q = 5, zero_phase = False):\n    return scipy.signal.decimate(data, q, axis = 0, zero_phase = zero_phase)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:39.809853Z","iopub.execute_input":"2024-01-30T01:10:39.810252Z","iopub.status.idle":"2024-01-30T01:10:39.815876Z","shell.execute_reply.started":"2024-01-30T01:10:39.810219Z","shell.execute_reply":"2024-01-30T01:10:39.814642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    print('Full shape', df_eeg_0.shape)\n    print('Downsampled shape', downsample(df_eeg_0).shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:40.609603Z","iopub.execute_input":"2024-01-30T01:10:40.609954Z","iopub.status.idle":"2024-01-30T01:10:40.620688Z","shell.execute_reply.started":"2024-01-30T01:10:40.609926Z","shell.execute_reply":"2024-01-30T01:10:40.619361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(pd.DataFrame(downsample(df_eeg_0), columns = df_eeg_0.columns), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:10:45.745248Z","iopub.execute_input":"2024-01-30T01:10:45.745641Z","iopub.status.idle":"2024-01-30T01:10:50.433378Z","shell.execute_reply.started":"2024-01-30T01:10:45.745610Z","shell.execute_reply":"2024-01-30T01:10:50.431543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.slice(6000, 2000).to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(pd.DataFrame(downsample(df_eeg_0)[1200:1600], columns = df_eeg_0.columns), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:56.013935Z","iopub.execute_input":"2024-01-30T01:09:56.014601Z","iopub.status.idle":"2024-01-30T01:09:58.696104Z","shell.execute_reply.started":"2024-01-30T01:09:56.014559Z","shell.execute_reply":"2024-01-30T01:09:58.693562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(pd.DataFrame(butter_lowpass_filter(df_eeg_0)[6000:8000], columns = df_eeg_0.columns), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    axes[0].set_title('Lowpass');\n    sns.lineplot(pd.DataFrame(downsample(df_eeg_0)[1200:1600], columns = df_eeg_0.columns), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));\n    axes[1].set_title('Downsample');","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:09:58.698038Z","iopub.execute_input":"2024-01-30T01:09:58.698375Z","iopub.status.idle":"2024-01-30T01:10:01.336514Z","shell.execute_reply.started":"2024-01-30T01:09:58.698344Z","shell.execute_reply":"2024-01-30T01:10:01.335201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Alternative way to downsample: averaging every 5 samples.","metadata":{}},{"cell_type":"code","source":"def downsample_pl(x, q = 5):\n    return (\n        x.with_row_count('row_count')\n        .group_by(pl.col('row_count') // q)\n        .mean()\n        .sort('row_count')\n        .drop('row_count')\n    )","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:12:42.308286Z","iopub.execute_input":"2024-01-30T01:12:42.308684Z","iopub.status.idle":"2024-01-30T01:12:42.314328Z","shell.execute_reply.started":"2024-01-30T01:12:42.308653Z","shell.execute_reply":"2024-01-30T01:12:42.313227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    display(downsample_pl(df_eeg_0))","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:13:01.075238Z","iopub.execute_input":"2024-01-30T01:13:01.075683Z","iopub.status.idle":"2024-01-30T01:13:01.128773Z","shell.execute_reply.started":"2024-01-30T01:13:01.075647Z","shell.execute_reply":"2024-01-30T01:13:01.127790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(downsample_pl(df_eeg_0).to_pandas(), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:13:43.024542Z","iopub.execute_input":"2024-01-30T01:13:43.024905Z","iopub.status.idle":"2024-01-30T01:13:47.736338Z","shell.execute_reply.started":"2024-01-30T01:13:43.024877Z","shell.execute_reply":"2024-01-30T01:13:47.735489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(df_eeg_0.slice(6000, 2000).to_pandas(), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    sns.lineplot(downsample_pl(df_eeg_0).slice(1200, 400).to_pandas(), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:16:12.401992Z","iopub.execute_input":"2024-01-30T01:16:12.403029Z","iopub.status.idle":"2024-01-30T01:16:15.015624Z","shell.execute_reply.started":"2024-01-30T01:16:12.402988Z","shell.execute_reply":"2024-01-30T01:16:15.014667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_main:\n    fig, axes = plt.subplots(2, figsize = (64, 16))\n    sns.lineplot(pd.DataFrame(downsample(df_eeg_0)[1200:1600], columns = df_eeg_0.columns), ax = axes[0]);\n    axes[0].set(ylim = (-1600, 300));\n    axes[0].set_title('SciPy downsample');\n    sns.lineplot(downsample_pl(df_eeg_0).slice(1200, 400).to_pandas(), ax = axes[1]);\n    axes[1].set(ylim = (-1600, 300));\n    axes[1].set_title('Polars downsample');","metadata":{"execution":{"iopub.status.busy":"2024-01-30T01:17:47.687387Z","iopub.execute_input":"2024-01-30T01:17:47.687749Z","iopub.status.idle":"2024-01-30T01:17:50.074212Z","shell.execute_reply.started":"2024-01-30T01:17:47.687721Z","shell.execute_reply":"2024-01-30T01:17:50.073110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}