{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:28:06.536751Z","iopub.execute_input":"2024-04-01T12:28:06.537173Z","iopub.status.idle":"2024-04-01T12:28:07.771853Z","shell.execute_reply.started":"2024-04-01T12:28:06.537138Z","shell.execute_reply":"2024-04-01T12:28:07.770757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nprint('Train shape', train.shape)\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:28:07.773661Z","iopub.execute_input":"2024-04-01T12:28:07.774100Z","iopub.status.idle":"2024-04-01T12:28:08.071446Z","shell.execute_reply.started":"2024-04-01T12:28:07.774072Z","shell.execute_reply":"2024-04-01T12:28:08.070241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handling Missing Values","metadata":{}},{"cell_type":"code","source":"L = os.listdir(\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\")","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:31:41.464310Z","iopub.execute_input":"2024-04-01T12:31:41.465035Z","iopub.status.idle":"2024-04-01T12:31:41.479907Z","shell.execute_reply.started":"2024-04-01T12:31:41.465000Z","shell.execute_reply":"2024-04-01T12:31:41.478782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_eeg_ids = [int(num[:-8]) for num in L if int(num[:-8]) not in train['eeg_id'].unique()]","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:31:41.709608Z","iopub.execute_input":"2024-04-01T12:31:41.709976Z","iopub.status.idle":"2024-04-01T12:32:02.396010Z","shell.execute_reply.started":"2024-04-01T12:31:41.709947Z","shell.execute_reply":"2024-04-01T12:32:02.394896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"There are {len(list(L))} file in the train_eegs which does not match the number of the unique eeg_id({train['eeg_id'].nunique()}) in train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:32:02.398275Z","iopub.execute_input":"2024-04-01T12:32:02.398950Z","iopub.status.idle":"2024-04-01T12:32:02.406890Z","shell.execute_reply.started":"2024-04-01T12:32:02.398910Z","shell.execute_reply":"2024-04-01T12:32:02.405874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handling Montages","metadata":{}},{"cell_type":"code","source":"NAMES = ['LL', 'LP', 'RL', 'RP']\n\nIMP_FEATS = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5','O1', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2']\n\nIMP_FEATS_index = {y:x for x,y in enumerate(IMP_FEATS)}\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:32:06.295961Z","iopub.execute_input":"2024-04-01T12:32:06.296320Z","iopub.status.idle":"2024-04-01T12:32:06.305142Z","shell.execute_reply.started":"2024-04-01T12:32:06.296294Z","shell.execute_reply":"2024-04-01T12:32:06.304188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Band Pass filter (0.5Hz - 20Hz)","metadata":{}},{"cell_type":"code","source":"from scipy.signal import butter, filtfilt\n\ndef butter_bandpass_filter(signal, lowcut, highcut, fs, order=4):\n    nyq = 0.5 * fs\n    low = lowcut / nyq\n    high = highcut / nyq\n    b, a = butter(order, [low, high], btype='band')\n    y = filtfilt(b, a, signal)\n    return y","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Scalograms","metadata":{}},{"cell_type":"markdown","source":"- Form the double banana Montages (LL, LP, RL, RP)\n- Take the middle 50 second from the parquet path and apply CWT to obtain the scalogram for each montage to form a variable containing 4 images.\n","metadata":{}},{"cell_type":"code","source":"import pywt\nfrom IPython.display import clear_output\ntime = np.linspace(0, 50, 10000)\nsampling_period = np.diff(time).mean()\ndef scalogram_from_eeg(parquet_path, wavelet='morl', scales = np.arange(1, 128)):\n\n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n\n    images = np.zeros((229,232,4),dtype='float32')\n    for k in range(4):\n        COLS = FEATS[k]\n        coeff = []\n        sum_of_coeff = np.zeros((127,len(eeg)))\n        for kk in range(4):\n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            nan_indices = np.isnan(x)\n            x[nan_indices] = np.nanmean(x)\n\n            # Denoise\n            x = butter_bandpass_filter(x, 0.5, 20, 200, order = 4)\n\n            coefficients, freqs = pywt.cwt(x, scales, wavelet)\n            coeff.append(np.abs(coefficients))\n\n        coeff = np.array(coeff)\n        sum_of_coeff = np.sum(coeff, axis = 0)\n        avg_coeff = sum_of_coeff / 4\n\n        fig, ax = plt.subplots(figsize=(3, 3))\n        ax.imshow(np.abs(avg_coeff), extent=[0, 1, 1, 128], cmap='jet', aspect='auto')\n        plt.savefig('scalogram.png')\n        plt.show()\n        image_array = plt.imread('scalogram.png')\n        images[:,:,k] = np.mean(image_array[37:266,39:271,:3]*[0.8, 0.1, 0.1],axis=2)\n        plt.close()\n        clear_output(wait=True)\n\n    return images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scalogram formation","metadata":{}},{"cell_type":"code","source":"# Function to save dictionary to .npy file\ndef save_dict_to_npy(dictionary, file_path):\n    np.save(file_path, dictionary)","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:32:15.158170Z","iopub.execute_input":"2024-04-01T12:32:15.159144Z","iopub.status.idle":"2024-04-01T12:32:15.164959Z","shell.execute_reply.started":"2024-04-01T12:32:15.159092Z","shell.execute_reply":"2024-04-01T12:32:15.163728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize the Scalogram dictionary\nnew_eeg = {}","metadata":{"execution":{"iopub.status.busy":"2024-04-01T12:38:44.155110Z","iopub.execute_input":"2024-04-01T12:38:44.155572Z","iopub.status.idle":"2024-04-01T12:38:44.160756Z","shell.execute_reply.started":"2024-04-01T12:38:44.155537Z","shell.execute_reply":"2024-04-01T12:38:44.159590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nEEG_IDS = train.eeg_id.unique()\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\nfile_path = '/kaggle/working/eeg_scalogram.npy'\nfor i, eeg_id in enumerate(EEG_IDS):    \n    \n    if eeg_id in missing_eeg_ids: \n        continue\n    else:\n        \n        if (i%1000==0) & (i>0):\n            save_dict_to_npy(new_eeg, file_path)\n\n        img = scalogram_from_eeg(f'{PATH}{eeg_id}.parquet')\n\n        new_eeg[eeg_id] = img\n    \nsave_dict_to_npy(new_eeg, file_path)","metadata":{"_kg_hide-output":false,"scrolled":true,"execution":{"iopub.status.busy":"2024-04-01T12:39:40.435746Z","iopub.execute_input":"2024-04-01T12:39:40.436155Z","iopub.status.idle":"2024-04-01T14:40:16.855435Z","shell.execute_reply.started":"2024-04-01T12:39:40.436113Z","shell.execute_reply":"2024-04-01T14:40:16.854107Z"},"trusted":true},"execution_count":null,"outputs":[]}]}