{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-07T13:04:50.290629Z","iopub.execute_input":"2024-03-07T13:04:50.291038Z","iopub.status.idle":"2024-03-07T13:04:51.532404Z","shell.execute_reply.started":"2024-03-07T13:04:50.291010Z","shell.execute_reply":"2024-03-07T13:04:51.530653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\ntest_df  = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\nsample_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:51.534648Z","iopub.execute_input":"2024-03-07T13:04:51.535305Z","iopub.status.idle":"2024-03-07T13:04:51.877531Z","shell.execute_reply.started":"2024-03-07T13:04:51.535267Z","shell.execute_reply":"2024-03-07T13:04:51.875840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:51.879976Z","iopub.execute_input":"2024-03-07T13:04:51.880419Z","iopub.status.idle":"2024-03-07T13:04:51.910229Z","shell.execute_reply.started":"2024-03-07T13:04:51.880389Z","shell.execute_reply":"2024-03-07T13:04:51.908632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:51.912922Z","iopub.execute_input":"2024-03-07T13:04:51.913299Z","iopub.status.idle":"2024-03-07T13:04:51.924582Z","shell.execute_reply.started":"2024-03-07T13:04:51.913267Z","shell.execute_reply":"2024-03-07T13:04:51.923493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"expert_consensus\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:51.926292Z","iopub.execute_input":"2024-03-07T13:04:51.926955Z","iopub.status.idle":"2024-03-07T13:04:51.967560Z","shell.execute_reply.started":"2024-03-07T13:04:51.926916Z","shell.execute_reply":"2024-03-07T13:04:51.966210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote_cnames = ['seizure_vote', 'lpd_vote',\n       'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n\ntotal_vote = np.sum(train_df[vote_cnames], axis =1).to_numpy()\nvote_mat = train_df[vote_cnames].to_numpy().astype(float)\nmax_vote_rate = np.max(vote_mat/(total_vote.reshape(-1,1)), axis = 1)\n\nfilter1 = max_vote_rate >= 1.\nfilter2 = total_vote >= 3\nfilter_confident = (filter1 & filter2)\n\ntrain_df.loc[filter_confident][\"expert_consensus\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:51.969210Z","iopub.execute_input":"2024-03-07T13:04:51.970528Z","iopub.status.idle":"2024-03-07T13:04:52.049253Z","shell.execute_reply.started":"2024-03-07T13:04:51.970475Z","shell.execute_reply":"2024-03-07T13:04:52.047668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_eegs_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/\"\ntrain_eegs_files = os.listdir(train_eegs_dir)\nlen(train_eegs_files)","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:52.050892Z","iopub.execute_input":"2024-03-07T13:04:52.051281Z","iopub.status.idle":"2024-03-07T13:04:52.186066Z","shell.execute_reply.started":"2024-03-07T13:04:52.051250Z","shell.execute_reply":"2024-03-07T13:04:52.185190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_spectro_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/\"\ntrain_spectro_files = os.listdir(train_spectro_dir)\nlen(train_spectro_files)","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:52.452847Z","iopub.execute_input":"2024-03-07T13:04:52.453663Z","iopub.status.idle":"2024-03-07T13:04:52.596091Z","shell.execute_reply.started":"2024-03-07T13:04:52.453627Z","shell.execute_reply":"2024-03-07T13:04:52.594969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readSpectrogram(fileID, test = False):\n    fid = str(fileID)\n    \n    if test:\n        spectro_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\"\n        data = pd.read_parquet(spectro_dir + fid + \".parquet\")\n        \n    else:\n        spectro_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/\"\n        data = pd.read_parquet(spectro_dir + fid + \".parquet\")\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:52.871301Z","iopub.execute_input":"2024-03-07T13:04:52.871950Z","iopub.status.idle":"2024-03-07T13:04:52.877975Z","shell.execute_reply.started":"2024-03-07T13:04:52.871917Z","shell.execute_reply":"2024-03-07T13:04:52.876992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readEEG(fileID, test = False):\n    \n    fid = str(fileID)\n    \n    if test:\n        egg_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\"\n        data = pd.read_parquet(egg_dir + fid + \".parquet\")\n    else:\n        egg_dir = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/\"\n        data = pd.read_parquet(egg_dir + fid + \".parquet\")\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:08:02.726197Z","iopub.execute_input":"2024-03-07T13:08:02.726599Z","iopub.status.idle":"2024-03-07T13:08:02.734428Z","shell.execute_reply.started":"2024-03-07T13:08:02.726559Z","shell.execute_reply":"2024-03-07T13:08:02.732979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spectrogram of Test Data","metadata":{}},{"cell_type":"code","source":"test_spect = readSpectrogram(test_df[\"spectrogram_id\"].iloc[0], True)\ntest_spect.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:04:59.528218Z","iopub.execute_input":"2024-03-07T13:04:59.528600Z","iopub.status.idle":"2024-03-07T13:04:59.732034Z","shell.execute_reply.started":"2024-03-07T13:04:59.528573Z","shell.execute_reply":"2024-03-07T13:04:59.731202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_spect.tail()","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:05:00.938511Z","iopub.execute_input":"2024-03-07T13:05:00.938919Z","iopub.status.idle":"2024-03-07T13:05:00.975933Z","shell.execute_reply.started":"2024-03-07T13:05:00.938887Z","shell.execute_reply":"2024-03-07T13:05:00.974777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spectrogram Data\n\nUnlike test data, length of spectrogram is not the same in train dataset.","metadata":{}},{"cell_type":"code","source":"for i in range(20):\n    idx = np.random.randint(train_df.shape[0])\n    spect_id = train_df[\"spectrogram_id\"].iloc[idx]\n    train_spect = readSpectrogram(spect_id)\n    print(train_spect.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-25T15:59:32.028220Z","iopub.execute_input":"2024-02-25T15:59:32.028790Z","iopub.status.idle":"2024-02-25T15:59:33.205474Z","shell.execute_reply.started":"2024-02-25T15:59:32.028738Z","shell.execute_reply":"2024-02-25T15:59:33.204419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time1 = time.time()\nfid_list = []\nlength_list = []\nfor fname in train_spectro_files:\n    fid = fname.split(\".\")[0]\n    train_spect = readSpectrogram(fid)\n    \n    fid_list.append(fid)\n    length_list.append(train_spect.shape[0])\n    \ntime2 = time.time()\ntime3 = np.round(time2 - time1)\nprint(time3, \"sec\")","metadata":{"execution":{"iopub.status.busy":"2024-02-25T15:59:57.555107Z","iopub.execute_input":"2024-02-25T15:59:57.555519Z","iopub.status.idle":"2024-02-25T16:00:02.587305Z","shell.execute_reply.started":"2024-02-25T15:59:57.555487Z","shell.execute_reply":"2024-02-25T16:00:02.585271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spect_df = pd.DataFrame({\"spectrogram_id\": fid_list, \"length\": length_list})","metadata":{"execution":{"iopub.status.busy":"2024-02-25T16:00:08.548046Z","iopub.execute_input":"2024-02-25T16:00:08.548568Z","iopub.status.idle":"2024-02-25T16:00:08.555378Z","shell.execute_reply.started":"2024-02-25T16:00:08.548530Z","shell.execute_reply":"2024-02-25T16:00:08.554070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spect_df.to_csv(\"spectrogram_length.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T16:00:08.741598Z","iopub.execute_input":"2024-02-25T16:00:08.742047Z","iopub.status.idle":"2024-02-25T16:00:08.751255Z","shell.execute_reply.started":"2024-02-25T16:00:08.742009Z","shell.execute_reply":"2024-02-25T16:00:08.750158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spect_df[\"length\"].hist(bins = 50)","metadata":{"execution":{"iopub.status.busy":"2024-02-25T16:01:03.724515Z","iopub.execute_input":"2024-02-25T16:01:03.724935Z","iopub.status.idle":"2024-02-25T16:01:04.031493Z","shell.execute_reply.started":"2024-02-25T16:01:03.724902Z","shell.execute_reply":"2024-02-25T16:01:04.030254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EEG Data\n\nSame for EEG","metadata":{}},{"cell_type":"code","source":"time1 = time.time()\nfid_list = []\nlength_list = []\n\nfor eeg_id in train_eegs_files:\n    eeg_id = eeg_id.split(\".\")[0]\n    eeg_data = readEEG(eeg_id)\n\n    fid_list.append(eeg_id)\n    length_list.append(eeg_data.shape[0])\n\n    \ntime2 = time.time()\ntime3 = np.round(time2 - time1)\nprint(time3, \"sec\")","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:21:04.772328Z","iopub.execute_input":"2024-03-07T13:21:04.772721Z","iopub.status.idle":"2024-03-07T13:21:05.723391Z","shell.execute_reply.started":"2024-03-07T13:21:04.772692Z","shell.execute_reply":"2024-03-07T13:21:05.721661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_df = pd.DataFrame({\"eeg_id\": fid_list, \"length\": length_list})","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:21:05.726062Z","iopub.execute_input":"2024-03-07T13:21:05.726523Z","iopub.status.idle":"2024-03-07T13:21:05.736671Z","shell.execute_reply.started":"2024-03-07T13:21:05.726483Z","shell.execute_reply":"2024-03-07T13:21:05.733309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_df.to_csv(\"eeg_length.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:21:05.738206Z","iopub.execute_input":"2024-03-07T13:21:05.738677Z","iopub.status.idle":"2024-03-07T13:21:05.747748Z","shell.execute_reply.started":"2024-03-07T13:21:05.738635Z","shell.execute_reply":"2024-03-07T13:21:05.746607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_df[\"length\"].hist(bins = 50)","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:21:05.899667Z","iopub.execute_input":"2024-03-07T13:21:05.900065Z","iopub.status.idle":"2024-03-07T13:21:06.204840Z","shell.execute_reply.started":"2024-03-07T13:21:05.900034Z","shell.execute_reply":"2024-03-07T13:21:06.203541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_df","metadata":{"execution":{"iopub.status.busy":"2024-03-07T13:21:08.576040Z","iopub.execute_input":"2024-03-07T13:21:08.577384Z","iopub.status.idle":"2024-03-07T13:21:08.589604Z","shell.execute_reply.started":"2024-03-07T13:21:08.577346Z","shell.execute_reply":"2024-03-07T13:21:08.588308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}