{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7680946,"sourceType":"datasetVersion","datasetId":4476546}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <div style=\"padding: 25px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#A51C30\"><b><span style='color:#FFFFFF'></span></b> <b>INTRODUCTION</b></div>\n\n📌 **From this NoteBook**: https://www.kaggle.com/code/seanbearden/processing-the-tuh-eeg-seizure-corpus\n\nThanks for sharing Online Dataset! `Sean R.B. Bearden, Ph.D.`\n\n\n📌 **Check Out**\n<div style=\"border-radius:10px; border: #babab5 solid; padding: 15px; background-color:##A51C30; font-size:100%;\">\n\n- This Dataset is just including seizure patients\n    \n- There're no votes(Pseudo Labeling is probably good option) \n    \n- Drop missing values > 0.4\n    \n- EEG_ID: 3050 -> 2208\n    \n- If you want to adding online dataset below\n    https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/471439\n    ","metadata":{}},{"cell_type":"markdown","source":"# <div style=\"padding: 25px;color:white;margin:10;font-size:60%;text-align:left;display:fill;border-radius:10px;background-color:#FFFFFF;overflow:hidden;background-color:#A51C30\"><b><span style='color:#FFFFFF'></span></b> <b>Dataset</b></div>\n\n📌 **TUH_10m_Spectrograms**: https://www.kaggle.com/datasets/seoyunje/tuh-10m-spectrogram","metadata":{}},{"cell_type":"code","source":"VER=2\n\nimport pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt, gc\nimport seaborn as sns\n\nfrom glob import glob\n\ntuh = pd.read_csv('/kaggle/input/hms-hba-tuh-tusz-seizures/seizures.csv')\nprint(tuh.shape)\ndisplay(tuh)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:45.364593Z","iopub.execute_input":"2024-03-09T00:35:45.364995Z","iopub.status.idle":"2024-03-09T00:35:48.545096Z","shell.execute_reply.started":"2024-03-09T00:35:45.364964Z","shell.execute_reply":"2024-03-09T00:35:48.543879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '/kaggle/input/hms-hba-tuh-tusz-seizures/eeg_raw/'\n\nfiles= glob(f'{PATH}*.parquet')\n\nfiles_id = []\n\nfor file in files:\n    id = file.split('/')[-1].split('.')[0]\n    files_id.append(id) \n    \ntuh = tuh[tuh['id'].isin(files_id)]\ntuh.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:48.547318Z","iopub.execute_input":"2024-03-09T00:35:48.548289Z","iopub.status.idle":"2024-03-09T00:35:48.870601Z","shell.execute_reply.started":"2024-03-09T00:35:48.548253Z","shell.execute_reply":"2024-03-09T00:35:48.869470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop = tuh['missing_data'] == 0\n\ntuh = tuh.loc[~drop]\nprint(f'There are {len(tuh)} train data')","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:48.877706Z","iopub.execute_input":"2024-03-09T00:35:48.878522Z","iopub.status.idle":"2024-03-09T00:35:48.889130Z","shell.execute_reply.started":"2024-03-09T00:35:48.878483Z","shell.execute_reply":"2024-03-09T00:35:48.887952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.histplot(data=tuh, x='missing_data', bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:48.890752Z","iopub.execute_input":"2024-03-09T00:35:48.891424Z","iopub.status.idle":"2024-03-09T00:35:49.355183Z","shell.execute_reply.started":"2024-03-09T00:35:48.891386Z","shell.execute_reply":"2024-03-09T00:35:49.354047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuh = tuh.drop(columns=['path','reference_type','group','session','class_code','directory','event_time','EKG','processed','missing_data'])","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.356792Z","iopub.execute_input":"2024-03-09T00:35:49.358105Z","iopub.status.idle":"2024-03-09T00:35:49.368606Z","shell.execute_reply.started":"2024-03-09T00:35:49.358037Z","shell.execute_reply":"2024-03-09T00:35:49.367500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuh.rename(columns={'id':'eeg_id'}, inplace=True)\ntuh.rename(columns={'start_time':'min', 'stop_time':'max'}, inplace=True)\n\n\ntmp = tuh['eeg_id'].values\ntuh['spec_id'] = tmp\n\ntmp = np.zeros((len(tuh),6))\nTARGETS = ['seizure_vote', 'lpd_vote','gpd_vote','lrda_vote','grda_vote','other_vote']\ntuh[TARGETS] = tmp\n\ntuh['target'] = 'Seizure'\n\nprint(tuh.shape)\ntuh.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.370391Z","iopub.execute_input":"2024-03-09T00:35:49.371114Z","iopub.status.idle":"2024-03-09T00:35:49.406438Z","shell.execute_reply.started":"2024-03-09T00:35:49.371080Z","shell.execute_reply":"2024-03-09T00:35:49.405171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NAMES_ADD = ['LL','LP','RP','RR']\n\nFEATS_ADD = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.408246Z","iopub.execute_input":"2024-03-09T00:35:49.408730Z","iopub.status.idle":"2024-03-09T00:35:49.416618Z","shell.execute_reply.started":"2024-03-09T00:35:49.408691Z","shell.execute_reply":"2024-03-09T00:35:49.415273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt\nprint(\"The wavelet functions we can use:\")\nprint(pywt.wavelist())\n\nUSE_WAVELET = None","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.418861Z","iopub.execute_input":"2024-03-09T00:35:49.419643Z","iopub.status.idle":"2024-03-09T00:35:49.519767Z","shell.execute_reply.started":"2024-03-09T00:35:49.419602Z","shell.execute_reply":"2024-03-09T00:35:49.518626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.523320Z","iopub.execute_input":"2024-03-09T00:35:49.524187Z","iopub.status.idle":"2024-03-09T00:35:49.532674Z","shell.execute_reply.started":"2024-03-09T00:35:49.524153Z","shell.execute_reply":"2024-03-09T00:35:49.531526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS_ADD[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n            \n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(3,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram_Add_N {NAMES_ADD[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES_ADD[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img    ","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.534211Z","iopub.execute_input":"2024-03-09T00:35:49.534590Z","iopub.status.idle":"2024-03-09T00:35:49.568446Z","shell.execute_reply.started":"2024-03-09T00:35:49.534540Z","shell.execute_reply":"2024-03-09T00:35:49.567242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPATH = '/kaggle/input/hms-hba-tuh-tusz-seizures/eeg_raw/'\nDISPLAY = 4\nEEG_IDS = tuh.eeg_id.unique()\nall_eegs = {}\n\nfor i,eeg_id in enumerate(EEG_IDS):\n    if (i%100==0)&(i!=0): print(i,', ',end='')\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH}{eeg_id}.parquet', i<DISPLAY)\n    \n    # SAVE TO DISK\n    if i==DISPLAY:\n        print(f'Creating and writing {len(EEG_IDS)} spectrograms to disk... ',end='')\n    all_eegs[eeg_id] = img\n   \n# SAVE EEG SPECTROGRAM DICTIONARY\nnp.save('eeg_specs',all_eegs)","metadata":{"execution":{"iopub.status.busy":"2024-03-09T00:35:49.569899Z","iopub.execute_input":"2024-03-09T00:35:49.570235Z"},"trusted":true},"execution_count":null,"outputs":[]}]}