{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 5th place solution of HMS competition - D.Imanishi's part\nThis notebook generates graph images (.png files) from raw eeg.\n#### Other notebooks of my solution\n- [HMS graph model training](https://www.kaggle.com/code/dimanishi/hms-graph-model-training)\n- [HMS graph model inference](https://www.kaggle.com/code/dimanishi/hms-graph-model-inference)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nfrom tqdm import tqdm\nimport glob\nimport matplotlib.pyplot as plt\nimport io\nimport cv2\nimport os\nfrom multiprocessing import Pool\nimport random\nfrom scipy.signal import butter, lfilter\nfrom IPython.display import FileLink\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-12T13:16:44.862902Z","iopub.execute_input":"2024-04-12T13:16:44.863401Z","iopub.status.idle":"2024-04-12T13:16:48.219688Z","shell.execute_reply.started":"2024-04-12T13:16:44.863364Z","shell.execute_reply":"2024-04-12T13:16:48.218291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_COLS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nEEG_DIR = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs'\n\nLL_PAIRS = [\n    ['Fp1', 'F7'],\n    ['F7', 'T3'],\n    ['T3', 'T5'],\n    ['T5', 'O1'],\n]\nRL_PAIRS = [\n    ['Fp2', 'F8'],\n    ['F8', 'T4'],\n    ['T4', 'T6'],\n    ['T6', 'O2'],\n]\nLP_PAIRS = [\n    ['Fp1', 'F3'],\n    ['F3', 'C3'],\n    ['C3', 'P3'],\n    ['P3', 'O1'],\n]\nRP_PAIRS = [\n    ['Fp2', 'F4'],\n    ['F4', 'C4'],\n    ['C4', 'P4'],\n    ['P4', 'O2'],\n]\n\nEEG_SAMPLE_SEC = 200\nEEG_TIMESPAN = 20.\nCUT_OFF = 20\nCLIP = 400\nV_OFFSET = 500\nLW = 0.5\n\nGRAPH_SIZE = (512, 1024) # height, width\nEDGE_CROP = 12\n\nSAVE_DIR = '/graphs'\nos.makedirs(f'{SAVE_DIR}/raw', exist_ok=True)\nos.makedirs(f'{SAVE_DIR}/lpf', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:16:58.064314Z","iopub.execute_input":"2024-04-12T13:16:58.064854Z","iopub.status.idle":"2024-04-12T13:16:58.077950Z","shell.execute_reply.started":"2024-04-12T13:16:58.064824Z","shell.execute_reply":"2024-04-12T13:16:58.076545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare color of 19 signals\ncmap = plt.get_cmap('jet', 19)\ncmap = [cmap(i) for i in range(19)]\nrandom.seed(0)\nrandom.shuffle(cmap)","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:16:59.174545Z","iopub.execute_input":"2024-04-12T13:16:59.174959Z","iopub.status.idle":"2024-04-12T13:16:59.189726Z","shell.execute_reply.started":"2024-04-12T13:16:59.174915Z","shell.execute_reply":"2024-04-12T13:16:59.188308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def butter_lowpass_filter(data, cutoff_freq=CUT_OFF, order=5):\n    nyquist = 0.5 * EEG_SAMPLE_SEC\n    normal_cutoff = cutoff_freq / nyquist\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    filtered_data = lfilter(b, a, data, axis=0)\n    return filtered_data\n\n\ndef clip_data(arr, is_negative=False, use_lpf=False):\n    if use_lpf:\n        if np.isnan(arr).sum() > 0:\n            arr = np.nan_to_num(arr, nan=np.nanmean(arr))\n        arr = butter_lowpass_filter(arr)\n    \n    arr = np.clip(arr, -CLIP, CLIP)\n    if is_negative:\n        arr = -arr\n    return arr\n\n\ndef draw_wave_graph(eeg_df_part, is_lrflip=False, is_negative=False, use_lpf=False):\n    if is_lrflip:\n        pairs = RL_PAIRS + LL_PAIRS + RP_PAIRS + LP_PAIRS\n    else:\n        pairs = LL_PAIRS + RL_PAIRS + LP_PAIRS + RP_PAIRS\n    \n    plt.figure(\n        figsize=((GRAPH_SIZE[1]+EDGE_CROP*2)/100, (GRAPH_SIZE[0]+EDGE_CROP*2)/100),\n        tight_layout=True)\n    \n    for i, pair in enumerate(pairs):\n        arr = clip_data(\n            eeg_df_part[pair[0]].values - eeg_df_part[pair[1]].values,\n            is_negative=is_negative, use_lpf=use_lpf)\n        plt.plot(arr - i*V_OFFSET, lw=LW, color=cmap[i])\n    \n    arr = clip_data(\n        eeg_df_part['Fz'].values - eeg_df_part['Cz'].values,\n        is_negative=is_negative, use_lpf=use_lpf)\n    plt.plot(arr - (i+1)*V_OFFSET, lw=LW, color=cmap[i+1])\n\n    arr = clip_data(\n        eeg_df_part['Cz'].values - eeg_df_part['Pz'].values,\n        is_negative=is_negative, use_lpf=use_lpf)\n    plt.plot(arr - (i+2)*V_OFFSET, lw=LW, color=cmap[i+2])\n\n    arr = clip_data(\n        eeg_df_part['EKG'].values,\n        is_negative=is_negative, use_lpf=use_lpf)\n    plt.plot(arr - (i+3)*V_OFFSET, lw=LW, color=cmap[i+3])\n\n    plt.xlim(0, EEG_SAMPLE_SEC*EEG_TIMESPAN)\n    plt.ylim(int(-19*V_OFFSET), int(0.5*V_OFFSET))\n    plt.axis('off')\n\n    buf = io.BytesIO()\n    plt.savefig(buf, format=\"png\")\n    buf.seek(0)\n    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)\n    buf.close()\n    img = cv2.imdecode(img_arr, 1)\n    plt.clf()\n    plt.close()\n    return img[EDGE_CROP:-EDGE_CROP, EDGE_CROP:-EDGE_CROP, :]\n\n\ndef generate_graph_files(eeg_id):\n    part_df = data_df[data_df['eeg_id']==eeg_id]\n    eeg_df = pd.read_parquet(f'{EEG_DIR}/{eeg_id}.parquet')\n\n    for i, ofs_time in enumerate(part_df['eeg_label_offset_seconds']):\n        start_idx = int(ofs_time * EEG_SAMPLE_SEC)\n        timespan_start_idx = start_idx + int((25 - EEG_TIMESPAN/2)*EEG_SAMPLE_SEC)\n        timespan_end_idx = start_idx + int((25 + EEG_TIMESPAN/2)*EEG_SAMPLE_SEC) - 1\n        eeg_df_part = eeg_df.loc[timespan_start_idx:timespan_end_idx].copy()\n        \n        lpf_ptns = [\n            dict(dir_path=f'{SAVE_DIR}/raw', use_lpf=False),\n            dict(dir_path=f'{SAVE_DIR}/lpf', use_lpf=True),\n        ]\n        for ptn in lpf_ptns:\n            use_lpf = ptn['use_lpf']\n            dir_path = ptn['dir_path']\n            \n            img = draw_wave_graph(eeg_df_part, is_lrflip=False, is_negative=False, use_lpf=use_lpf)\n            cv2.imwrite(f'{dir_path}/{eeg_id}_{i}.png', img)\n\n            # for augmentation\n            img = draw_wave_graph(eeg_df_part, is_lrflip=True, is_negative=False, use_lpf=use_lpf)\n            cv2.imwrite(f'{dir_path}/{eeg_id}_{i}_lrflip.png', img)\n\n            img = draw_wave_graph(eeg_df_part, is_lrflip=False, is_negative=True, use_lpf=use_lpf)\n            cv2.imwrite(f'{dir_path}/{eeg_id}_{i}_neg.png', img)\n\n            img = draw_wave_graph(eeg_df_part, is_lrflip=True, is_negative=True, use_lpf=use_lpf)\n            cv2.imwrite(f'{dir_path}/{eeg_id}_{i}_lrflip_neg.png', img)\n    return None","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:17:00.529569Z","iopub.execute_input":"2024-04-12T13:17:00.530103Z","iopub.status.idle":"2024-04-12T13:17:00.556383Z","shell.execute_reply.started":"2024-04-12T13:17:00.530058Z","shell.execute_reply":"2024-04-12T13:17:00.555302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data sampling\ndata_df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n\ndata_df['n_votes'] = data_df[TARGET_COLS].sum(axis=1)\n\nidxs = []\nfor _, df in tqdm(data_df.groupby('eeg_id')):\n    if len(df['expert_consensus'].unique()) > 1:\n        for _, sdf in df.groupby('expert_consensus'):\n            df2 = sdf[sdf['n_votes']==sdf['n_votes'].max()]\n            idxs.append(df2.index[len(df2) // 2])\n    else:\n        df2 = df[df['n_votes']==df['n_votes'].max()]\n        idxs.append(df2.index[len(df2) // 2])\n        \ndata_df = data_df[data_df.index.isin(idxs)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:17:01.816535Z","iopub.execute_input":"2024-04-12T13:17:01.817411Z","iopub.status.idle":"2024-04-12T13:17:14.885312Z","shell.execute_reply.started":"2024-04-12T13:17:01.817376Z","shell.execute_reply":"2024-04-12T13:17:14.884114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check generated graph image\nfor index, row in data_df.sample(5).iterrows():\n    ofs_time = row['eeg_label_offset_seconds']\n    eeg_id = row['eeg_id']\n    eeg_df = pd.read_parquet(f'{EEG_DIR}/{eeg_id}.parquet')\n\n    start_idx = int(ofs_time * EEG_SAMPLE_SEC)\n    timespan_start_idx = start_idx + int((25 - EEG_TIMESPAN/2)*EEG_SAMPLE_SEC)\n    timespan_end_idx = start_idx + int((25 + EEG_TIMESPAN/2)*EEG_SAMPLE_SEC) - 1\n    eeg_df_part = eeg_df.loc[timespan_start_idx:timespan_end_idx].copy()\n    img = draw_wave_graph(eeg_df_part)\n    \n    plt.imshow(img)\n    plt.title(row['eeg_id'])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:17:18.917734Z","iopub.execute_input":"2024-04-12T13:17:18.918174Z","iopub.status.idle":"2024-04-12T13:17:22.113319Z","shell.execute_reply.started":"2024-04-12T13:17:18.918143Z","shell.execute_reply":"2024-04-12T13:17:22.111493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generation\nfor eeg_ids in tqdm(np.array_split(data_df['eeg_id'].unique(), 50)):\n    with Pool(4) as pool:\n        imap = pool.imap(generate_graph_files, eeg_ids)\n        res = list(imap)","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:17:25.405576Z","iopub.execute_input":"2024-04-12T13:17:25.405982Z","iopub.status.idle":"2024-04-12T13:20:26.839854Z","shell.execute_reply.started":"2024-04-12T13:17:25.405953Z","shell.execute_reply":"2024-04-12T13:20:26.838296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export as zip file\nzip_file = f'./{os.path.basename(SAVE_DIR)}.zip'\n!zip -rq {zip_file} {SAVE_DIR}\ndisplay(FileLink(zip_file))","metadata":{"execution":{"iopub.status.busy":"2024-04-12T13:20:26.843044Z","iopub.execute_input":"2024-04-12T13:20:26.843505Z","iopub.status.idle":"2024-04-12T13:20:32.749968Z","shell.execute_reply.started":"2024-04-12T13:20:26.843468Z","shell.execute_reply":"2024-04-12T13:20:32.748378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}