{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🎨 dataset [here](https://www.kaggle.com/datasets/nakagawaren0805/hms-preprocessed-dataset)\n\nPreprocessing HMS Dataset 🚀\n===\n## content 🧠\n- denoise with demy(db8) 🌊\n- fill missing value with linear interpolate 📈\n---\n## output 📦\n- preprocessed Electroencephalogram (EEG) 🧠✨\n- preprocedded train.csv(discard still contain missing value eeg-id data) 📑🚫\n- preprocess function 🔄\n---\n## Lemma 📚\n- Test dataset don't have missing value: [check submission code](https://www.kaggle.com/code/nakagawaren0805/hms-detect-na) ✅\n- Use Group KFold to patient, in order to protect for data leakage! 🤝🔍","metadata":{}},{"cell_type":"markdown","source":"# Import modules","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pywt\nimport torch","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-23T12:51:05.794377Z","iopub.execute_input":"2024-01-23T12:51:05.795044Z","iopub.status.idle":"2024-01-23T12:51:09.665119Z","shell.execute_reply.started":"2024-01-23T12:51:05.794968Z","shell.execute_reply":"2024-01-23T12:51:09.664186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set parameters","metadata":{}},{"cell_type":"markdown","source":"## you don't need to change parameter","metadata":{}},{"cell_type":"code","source":"BASE_DIR = Path(\"/kaggle/input/hms-harmful-brain-activity-classification\")\nTRAIN_EEG_DIR = BASE_DIR/\"train_eegs\"\n\nEEG_SAMPLING_TIME = 50  #second\nEEG_SAMPLING_RATE = 200 #Hz\nEEG_DURATION = EEG_SAMPLING_RATE * EEG_SAMPLING_TIME\n\nN_INTERACTIVE = 100 # number of sampling for development environment","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:09.666581Z","iopub.execute_input":"2024-01-23T12:51:09.668390Z","iopub.status.idle":"2024-01-23T12:51:09.673542Z","shell.execute_reply.started":"2024-01-23T12:51:09.668359Z","shell.execute_reply":"2024-01-23T12:51:09.672564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## you can customize parameter","metadata":{}},{"cell_type":"code","source":"WAVELET_DECOMPOSITION_TREE_LEVEL = 1\nWAVELET_TYPE = \"dmey\"\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:09.674781Z","iopub.execute_input":"2024-01-23T12:51:09.675117Z","iopub.status.idle":"2024-01-23T12:51:09.682454Z","shell.execute_reply.started":"2024-01-23T12:51:09.675056Z","shell.execute_reply":"2024-01-23T12:51:09.681494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set a seed","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)  # 🎲 Set seed for NumPy\n    torch.manual_seed(seed)  # 🚀 Set seed for PyTorch on CPU\n    torch.cuda.manual_seed(seed)  # 🚀 Set seed for PyTorch on GPU\n    \n    # ⚙️ When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    # 🔏 Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n# 🌱 Set seed using the configured seed value\nset_seed()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:09.684723Z","iopub.execute_input":"2024-01-23T12:51:09.685508Z","iopub.status.idle":"2024-01-23T12:51:09.696808Z","shell.execute_reply.started":"2024-01-23T12:51:09.685479Z","shell.execute_reply":"2024-01-23T12:51:09.695984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define preprocess function","metadata":{}},{"cell_type":"code","source":"def plot_eeg(df, moving_avg=1):\n    fig, axs = plt.subplots(20, 1, figsize=(30, 15), sharex=True)\n    for i, ax in enumerate(axs):\n        ax.plot(df.iloc[:,i], color=\"black\")\n        for vline in df[df.iloc[:,i].isna()].index:\n            line_min = df.iloc[:,i].min()\n            line_max = df.iloc[:,i].max()\n            ax.vlines(vline, line_min, line_max, color='red')\n        ax.set_ylabel(df.columns[i], rotation=0)\n        ax.set_yticklabels([])\n        ax.set_yticks([])\n        ax.set_xticks([])\n        ax.spines[[\"top\", \"bottom\", \"left\", \"right\"]].set_visible(False)\n\ndef get_train_eeg(q:dict) -> pd.DataFrame:\n    parquet_df = pd.read_parquet(BASE_DIR/f\"train_eegs/{q['eeg_id']}.parquet\")\n    eeg_start_index = int(EEG_SAMPLING_RATE * q[\"eeg_label_offset_seconds\"])\n    return parquet_df.iloc[eeg_start_index:eeg_start_index+EEG_DURATION]\n\ndef denoise(x, wavelet='db8', level=1): # dmey for seizure patient denoise, db8 for healthy patient denoise\n    # paper: https://jart.icat.unam.mx/index.php/jart/article/view/339/336\n    def _maddest(d, axis=None):\n        return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n    ret = {key:[] for key in x.columns}\n    for pos in x.columns:\n        coeff = pywt.wavedec(x[pos], wavelet, mode=\"per\")\n        sigma = (1/0.6745) * _maddest(coeff[-level])\n        uthresh = sigma * np.sqrt(2*np.log(len(x)))\n        coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n        ret[pos]=pywt.waverec(coeff, wavelet, mode='per')\n    return pd.DataFrame(ret)\n\ndef interpolate(raw_df):\n    df = raw_df.copy()\n    df = df.interpolate(\n        method='linear',\n        axis=0,\n        limit=1, # ref to 1 value\n        limit_direction=\"both\", # interpolate from pre and post values\n        limit_area='inside',\n    )\n    return df\n\ndef replace_outlier(series, bias=1.5, upper=0.95, lower=0.05):\n    lower_clip = series.quantile(lower)\n    upper_clip = series.quantile(upper)\n    iqr = upper_clip - lower_clip\n\n    outlier_min = lower_clip - (iqr) * bias\n    outlier_max = upper_clip + (iqr) * bias\n\n    series = series.clip(outlier_min, outlier_max)\n    return series","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:09.698099Z","iopub.execute_input":"2024-01-23T12:51:09.698627Z","iopub.status.idle":"2024-01-23T12:51:09.716345Z","shell.execute_reply.started":"2024-01-23T12:51:09.698590Z","shell.execute_reply":"2024-01-23T12:51:09.715675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Note: Kaggle environment cannot make large file,so this is demonstration.  \n# If you want to obtain preprocessed data, you can get [here](https://www.kaggle.com/datasets/nakagawaren0805/hms-preprocessed-dataset)","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(BASE_DIR/\"train.csv\")\neeg_list = glob.glob(str(TRAIN_EEG_DIR/\"*.parquet\"))\nif True or os.environ.get('KAGGLE_KERNEL_RUN_TYPE','') == 'Interactive':\n    print(\"Running on development environment\")\n#     train_df = train_df.sample(N_INTERACTIVE)\n    eeg_list = eeg_list[:N_INTERACTIVE]\nelif os.environ.get('KAGGLE_KERNEL_RUN_TYPE','') == 'Batch':\n    print(\"Running on production environment\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:09.717390Z","iopub.execute_input":"2024-01-23T12:51:09.718401Z","iopub.status.idle":"2024-01-23T12:51:10.379951Z","shell.execute_reply.started":"2024-01-23T12:51:09.718364Z","shell.execute_reply":"2024-01-23T12:51:10.379133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess EEGs","metadata":{}},{"cell_type":"code","source":"preprocessed_eeg_dir = \"train_eegs\"\nos.makedirs(preprocessed_eeg_dir, exist_ok=True)\n\nignore_parquet_num = 0\nignore_parquet_eeg_ids = list()\nfor p_eeg in eeg_list:\n    eeg_id = int(p_eeg.split(\"/\")[-1].split(\".\")[0])\n    eeg = pd.read_parquet(p_eeg)\n    eeg = interpolate(eeg)\n    if eeg.isna().sum().sum(): # discard EEG(still contail missing value)\n        ignore_parquet_num+=1\n        ignore_parquet_eeg_ids.append(eeg_id)\n        continue\n    eeg = denoise(eeg, wavelet=WAVELET_TYPE)\n    eeg.to_parquet(os.path.join(preprocessed_eeg_dir, f\"{eeg_id}.parquet\"))\nprint(f\"{ignore_parquet_num=}/{len(eeg_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:10.381062Z","iopub.execute_input":"2024-01-23T12:51:10.381720Z","iopub.status.idle":"2024-01-23T12:51:24.034795Z","shell.execute_reply.started":"2024-01-23T12:51:10.381691Z","shell.execute_reply":"2024-01-23T12:51:24.032778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess train.csv","metadata":{}},{"cell_type":"code","source":"preprocessed_traindf = train_df.loc[~train_df[\"eeg_id\"].isin(ignore_parquet_eeg_ids)].copy()\npreprocessed_traindf.to_csv(\"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T12:51:24.036483Z","iopub.execute_input":"2024-01-23T12:51:24.036834Z","iopub.status.idle":"2024-01-23T12:51:24.938178Z","shell.execute_reply.started":"2024-01-23T12:51:24.036807Z","shell.execute_reply":"2024-01-23T12:51:24.937208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# That's all preprocessing.\n# You can use preprocessed dataset like original train dataset!\n# Tell me if you notice any problem, no matter how trivial!!","metadata":{}},{"cell_type":"markdown","source":"# Note: Kaggle environment cannot make large file,so this is demonstration.  \n# If you want to obtain preprocessed data, you can get [here](https://www.kaggle.com/datasets/nakagawaren0805/hms-preprocessed-dataset)","metadata":{}}]}