{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7415826,"sourceType":"datasetVersion","datasetId":4314033}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **aha-brain-eeg**\n**brain eeg classification** \n1. read csv & parquet","metadata":{}},{"cell_type":"code","source":"#INTERNET_ENABLED = False   # submission requires internet disabled\nINTERNET_ENABLED = True\n\nif INTERNET_ENABLED:\n    !pip install icecream","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:31:29.547183Z","iopub.execute_input":"2024-01-26T21:31:29.547454Z","iopub.status.idle":"2024-01-26T21:31:43.353460Z","shell.execute_reply.started":"2024-01-26T21:31:29.547429Z","shell.execute_reply":"2024-01-26T21:31:43.352334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    print(str(len(filenames)) + \" files in \" + dirname)\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Input data files are available in the read-only \"../input/\" directory\n\n# You can write up to 20GB to the current directory (/kaggle/working/) \n# that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:31:43.355604Z","iopub.execute_input":"2024-01-26T21:31:43.355995Z","iopub.status.idle":"2024-01-26T21:31:53.551560Z","shell.execute_reply.started":"2024-01-26T21:31:43.355961Z","shell.execute_reply":"2024-01-26T21:31:53.550592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport seaborn as sns\nimport pyarrow.parquet as pq\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:31:53.552662Z","iopub.execute_input":"2024-01-26T21:31:53.552956Z","iopub.status.idle":"2024-01-26T21:32:06.934239Z","shell.execute_reply.started":"2024-01-26T21:31:53.552913Z","shell.execute_reply":"2024-01-26T21:32:06.933220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if INTERNET_ENABLED:\n    from icecream import ic\n    ic(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:06.937099Z","iopub.execute_input":"2024-01-26T21:32:06.938106Z","iopub.status.idle":"2024-01-26T21:32:07.599342Z","shell.execute_reply.started":"2024-01-26T21:32:06.938075Z","shell.execute_reply":"2024-01-26T21:32:07.598463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append( \"/kaggle/input/ahapyutil-1a\" )\n# from tuner import *\n# tuner\n# ic(tuner)\nfrom stopwatch import *\nfrom stopwatch_unittest import *","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:07.600724Z","iopub.execute_input":"2024-01-26T21:32:07.601444Z","iopub.status.idle":"2024-01-26T21:32:07.623689Z","shell.execute_reply.started":"2024-01-26T21:32:07.601409Z","shell.execute_reply":"2024-01-26T21:32:07.622964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################\nfrom types import SimpleNamespace\n\ntuner = SimpleNamespace(\n\n    # input files paths\n    FILE_PATH_BASE=\"/kaggle/input/hms-harmful-brain-activity-classification/\",\n    TRAIN_CSV_FILENAME=\"train.csv\",\n    TEST_CSV_FILENAME=\"test.csv\",\n    TRAIN_EEG_PATH=\"train_eegs/\",\n    TEST_EEG_PATH=\"test_eegs/\",\n    TRAIN_SPECTROGRAM_PATH=\"train_spectrograms/\",\n    TEST_SPECTROGRAM_PATH=\"test_spectrograms/\",\n\n    PARQUET_FILE_EXT=\".parquet\",\n\n    # train csv column labels\n    EEG_ID_COLUMN = 'eeg_id',\n    EEG_SUB_ID_COLUMN = 'eeg_sub_id',\n    # eeg_label_offset_seconds \n    SPECTROGRAM_ID_COLUMN = \"spectrogram_id\",\n    SPECTROGRAM_SUB_ID_COLUMN = \"spectrogram_sub_id\",\n    # spectrogram_label_offset_seconds\n    LABEL_ID_COLUMN = \"label_id\",\n    PATIENT_ID_COLUMN = \"patient_id\",\n    EXPERT_CONSENSUS_COLUMN = \"expert_consensus\",\n    SEIZURE_VOTE_COLUMN = \"seizure_vote\",\n    LPD_VOTE_COLUMN = \"lpd_vote\",\n    GPD_VOTE_COLUMN = \"gpd_vote\",\n    LRDA_VOTE_COLUMN = \"lrda_vote\",\n    GRDA_VOTE_COLUMN = \"grda_vote\",\n    OTHER_VOTE_COLUMN = \"other_vote\"\n)\nif INTERNET_ENABLED:\n    ic(tuner)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:07.624713Z","iopub.execute_input":"2024-01-26T21:32:07.624986Z","iopub.status.idle":"2024-01-26T21:32:07.793045Z","shell.execute_reply.started":"2024-01-26T21:32:07.624962Z","shell.execute_reply":"2024-01-26T21:32:07.792183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#stopwatch_unittest()\nwatch = stopwatch()\nwatch.start()\nwatch.stop()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:07.794442Z","iopub.execute_input":"2024-01-26T21:32:07.794740Z","iopub.status.idle":"2024-01-26T21:32:07.799486Z","shell.execute_reply.started":"2024-01-26T21:32:07.794715Z","shell.execute_reply":"2024-01-26T21:32:07.798539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start a timer to measure \"run all\" duration\nwatch_run_all = stopwatch()\nwatch_run_all.start()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:07.800950Z","iopub.execute_input":"2024-01-26T21:32:07.801273Z","iopub.status.idle":"2024-01-26T21:32:07.809884Z","shell.execute_reply.started":"2024-01-26T21:32:07.801242Z","shell.execute_reply":"2024-01-26T21:32:07.808776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **train.csv, spectrogram, eeg**\n**train.csv** \n1. read\n**spectrogram** \n1. decode spectrogram id into file path\n2. read\n**eeg** \n1. decode eeg id into file path\n2. read","metadata":{}},{"cell_type":"code","source":"# read train.csv\n#train_csv_df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ntrain_csv_df = pd.read_csv(tuner.FILE_PATH_BASE + tuner.TRAIN_CSV_FILENAME)\n\n# show content snippet\npd.set_option('display.max_rows', 128)\ntrain_csv_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:07.811007Z","iopub.execute_input":"2024-01-26T21:32:07.811370Z","iopub.status.idle":"2024-01-26T21:32:08.104014Z","shell.execute_reply.started":"2024-01-26T21:32:07.811346Z","shell.execute_reply":"2024-01-26T21:32:08.103115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_csv_df['eeg_id'].dtype\ntrain_csv_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.107360Z","iopub.execute_input":"2024-01-26T21:32:08.107686Z","iopub.status.idle":"2024-01-26T21:32:08.115445Z","shell.execute_reply.started":"2024-01-26T21:32:08.107659Z","shell.execute_reply":"2024-01-26T21:32:08.114570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique eeg_id = 17089 of 106800 (# train eegs = 17270)\n#train_csv_df['eeg_id']\ntrain_csv_eeg_id_count = train_csv_df.shape[0]\nunique = train_csv_df['eeg_id'].unique()\n\ntrain_csv_unique_eeg_id_count = unique.shape[0]\nif INTERNET_ENABLED:\n    ic(train_csv_eeg_id_count, train_csv_unique_eeg_id_count)\n\ntrain_csv_ave_sub_id_count = train_csv_eeg_id_count / train_csv_unique_eeg_id_count\nif INTERNET_ENABLED:\n    ic(train_csv_ave_sub_id_count)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.116549Z","iopub.execute_input":"2024-01-26T21:32:08.116848Z","iopub.status.idle":"2024-01-26T21:32:08.187142Z","shell.execute_reply.started":"2024-01-26T21:32:08.116825Z","shell.execute_reply":"2024-01-26T21:32:08.186323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assign eeg_id_interest - first id\n#eeg_id_interest = 1628180742\n\neeg_id_interest = train_csv_df['eeg_id'].iloc[0]\neeg_id_interest","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.188146Z","iopub.execute_input":"2024-01-26T21:32:08.188378Z","iopub.status.idle":"2024-01-26T21:32:08.194295Z","shell.execute_reply.started":"2024-01-26T21:32:08.188357Z","shell.execute_reply":"2024-01-26T21:32:08.193392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"watch = stopwatch()\n\n# watch.start()\n# train_csv_iter_df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv', iterator=True, chunksize=1000)\n# train_csv_filtered_df = pd.concat([chunk[chunk['eeg_id'] == eeg_id_interest] for chunk in train_csv_iter_df])\n# watch.stop() # timer duration -> 0.01m\n\nwatch.start()\ntrain_csv_filtered_df = train_csv_df[(train_csv_df['eeg_id'] == eeg_id_interest)]\nwatch.stop() # timer duration -> 0.00m\n\ntrain_csv_filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.195379Z","iopub.execute_input":"2024-01-26T21:32:08.195679Z","iopub.status.idle":"2024-01-26T21:32:08.222446Z","shell.execute_reply.started":"2024-01-26T21:32:08.195649Z","shell.execute_reply":"2024-01-26T21:32:08.221502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_interest = train_csv_df[tuner.PATIENT_ID_COLUMN].iloc[0]\npatient_id_interest","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.223801Z","iopub.execute_input":"2024-01-26T21:32:08.224358Z","iopub.status.idle":"2024-01-26T21:32:08.231966Z","shell.execute_reply.started":"2024-01-26T21:32:08.224294Z","shell.execute_reply":"2024-01-26T21:32:08.231170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv_filtered_patient_df = train_csv_df[(train_csv_df[tuner.PATIENT_ID_COLUMN] == patient_id_interest)]\ntrain_csv_filtered_patient_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.232967Z","iopub.execute_input":"2024-01-26T21:32:08.233215Z","iopub.status.idle":"2024-01-26T21:32:08.269245Z","shell.execute_reply.started":"2024-01-26T21:32:08.233193Z","shell.execute_reply":"2024-01-26T21:32:08.268459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train spectrograms (parquet->panda)\n#parquet_spectro_filename = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/1000086677.parquet'\nspectro_id = train_csv_filtered_df[tuner.SPECTROGRAM_ID_COLUMN].iloc[0]\nif INTERNET_ENABLED:\n    ic(spectro_id)\nparquet_spectro_filename = tuner.FILE_PATH_BASE + tuner.TRAIN_SPECTROGRAM_PATH + str(spectro_id) + tuner.PARQUET_FILE_EXT\ntrain_spectro_df = pq.read_table(parquet_spectro_filename).to_pandas()\ntrain_spectro_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.270354Z","iopub.execute_input":"2024-01-26T21:32:08.270687Z","iopub.status.idle":"2024-01-26T21:32:08.467399Z","shell.execute_reply.started":"2024-01-26T21:32:08.270655Z","shell.execute_reply":"2024-01-26T21:32:08.466511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/353733.parquet","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:08.468529Z","iopub.execute_input":"2024-01-26T21:32:08.468861Z","iopub.status.idle":"2024-01-26T21:32:09.414333Z","shell.execute_reply.started":"2024-01-26T21:32:08.468824Z","shell.execute_reply":"2024-01-26T21:32:09.413097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train EEG (parquet->panda)\n#parquet_eeg_filename = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet'\neeg_id = train_csv_filtered_df[tuner.EEG_ID_COLUMN].iloc[0]\nif INTERNET_ENABLED:\n    ic(eeg_id)\nparquet_eeg_filename = tuner.FILE_PATH_BASE + tuner.TRAIN_EEG_PATH + str(eeg_id) + tuner.PARQUET_FILE_EXT\ntrain_eeg_pd = pq.read_table(parquet_eeg_filename).to_pandas()\ntrain_eeg_pd","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:09.416380Z","iopub.execute_input":"2024-01-26T21:32:09.417101Z","iopub.status.idle":"2024-01-26T21:32:09.519044Z","shell.execute_reply.started":"2024-01-26T21:32:09.417059Z","shell.execute_reply":"2024-01-26T21:32:09.518131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/input/hms-harmful-brain-activity-classification/train_eegs/162818*.parquet","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:09.520107Z","iopub.execute_input":"2024-01-26T21:32:09.520466Z","iopub.status.idle":"2024-01-26T21:32:10.470647Z","shell.execute_reply.started":"2024-01-26T21:32:09.520437Z","shell.execute_reply":"2024-01-26T21:32:10.469656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **test csv, spectrogram, eeg**\n**test.csv** \n1. read\n**spectrogram** \n1. decode spectrogram id into file path\n2. read\n**eeg** \n1. decode eeg id into file path\n2. read","metadata":{}},{"cell_type":"code","source":"# read test.csv\n#test_csv = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\ntest_csv_df = pd.read_csv(tuner.FILE_PATH_BASE + tuner.TEST_CSV_FILENAME)\n\n# show content snippet\npd.set_option('display.max_rows', 128)\ntest_csv_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:10.472512Z","iopub.execute_input":"2024-01-26T21:32:10.472845Z","iopub.status.idle":"2024-01-26T21:32:10.487811Z","shell.execute_reply.started":"2024-01-26T21:32:10.472815Z","shell.execute_reply":"2024-01-26T21:32:10.486889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test spectrograms (parquet->panda)\n#parquet_spectro_filename = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/853520.parquet'\nspectro_id = test_csv_df[tuner.SPECTROGRAM_ID_COLUMN].iloc[0]\nif INTERNET_ENABLED:\n    ic(spectro_id)\nparquet_spectro_filename = tuner.FILE_PATH_BASE + tuner.TEST_SPECTROGRAM_PATH + str(spectro_id) + tuner.PARQUET_FILE_EXT\ntest_spectro_df = pq.read_table(parquet_spectro_filename).to_pandas()\ntest_spectro_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:10.489169Z","iopub.execute_input":"2024-01-26T21:32:10.489651Z","iopub.status.idle":"2024-01-26T21:32:10.610294Z","shell.execute_reply.started":"2024-01-26T21:32:10.489598Z","shell.execute_reply":"2024-01-26T21:32:10.609398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test EEG (parquet->panda)\n#parquet_eeg_filename = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/3911565283.parquet'\neeg_id = test_csv_df[tuner.EEG_ID_COLUMN].iloc[0]\nif INTERNET_ENABLED:\n    ic(eeg_id)\nparquet_eeg_filename = tuner.FILE_PATH_BASE + tuner.TEST_EEG_PATH + str(eeg_id) + tuner.PARQUET_FILE_EXT\ntest_eeg_pd = pq.read_table(parquet_eeg_filename).to_pandas()\ntest_eeg_pd","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:10.611367Z","iopub.execute_input":"2024-01-26T21:32:10.611662Z","iopub.status.idle":"2024-01-26T21:32:10.688510Z","shell.execute_reply.started":"2024-01-26T21:32:10.611637Z","shell.execute_reply":"2024-01-26T21:32:10.687538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **train**\n**build model** \n1. \n**batch data** \n1. \n**train model** \n1. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **predict** \n1. for each \n        predict\n        update submission csv\n2. write submission","metadata":{}},{"cell_type":"markdown","source":"# **submission.csv sample format**\n\n### votes are probabilities - percentages\n### sum of votes = 1 (100%)\n     \n### eeg_id, seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote\n### 3911565283,0.16666666666666666,0.16666666666666666,0.16666666666666666,0.16666666666666666,0.16666666666666666,0.16666666666666666\n","metadata":{}},{"cell_type":"code","source":"dummy_data = []\nprediction_sixth = 1.0 / 6.0 \nif INTERNET_ENABLED:\n    ic(prediction_sixth)\nfor index, row in test_csv_df.iterrows():\n    eeg_id = row[tuner.EEG_ID_COLUMN]\n    if INTERNET_ENABLED:\n        ic(index, row)\n        ic(eeg_id)\n    dummy_data.append(\n        [eeg_id,\\\n        prediction_sixth,\\\n        prediction_sixth,\\\n        prediction_sixth,\\\n        prediction_sixth,\\\n        prediction_sixth,\\\n        prediction_sixth]\n    )\n\n# create panda DataFrame\ntest_submission_df = pd.DataFrame(dummy_data, \\\n                                  columns=[tuner.EEG_ID_COLUMN,\\\n                                           tuner.SEIZURE_VOTE_COLUMN,\\\n                                           tuner.LPD_VOTE_COLUMN,\\\n                                           tuner.GPD_VOTE_COLUMN,\\\n                                           tuner.LRDA_VOTE_COLUMN,\\\n                                           tuner.GRDA_VOTE_COLUMN,\\\n                                           tuner.OTHER_VOTE_COLUMN])\ntest_submission_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:10.689562Z","iopub.execute_input":"2024-01-26T21:32:10.689868Z","iopub.status.idle":"2024-01-26T21:32:10.811129Z","shell.execute_reply.started":"2024-01-26T21:32:10.689843Z","shell.execute_reply":"2024-01-26T21:32:10.810254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set submission.csv column labels\n#    seizure_vote,lpd_vote,gpd_vote,lrda_vote,grda_vote,other_vote\n\n# write series label lists to disk renaming as \"submission\"\n!pwd\ntest_submission_df.to_csv('/kaggle/working/submission.csv', index=False)\n!ls -l\n!pwd\n","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:10.812340Z","iopub.execute_input":"2024-01-26T21:32:10.812958Z","iopub.status.idle":"2024-01-26T21:32:13.680480Z","shell.execute_reply.started":"2024-01-26T21:32:10.812925Z","shell.execute_reply":"2024-01-26T21:32:13.679339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"aha_brain_eeg after running last cell->\")\nwatch_run_all.stop()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T21:32:13.682208Z","iopub.execute_input":"2024-01-26T21:32:13.682595Z","iopub.status.idle":"2024-01-26T21:32:13.688544Z","shell.execute_reply.started":"2024-01-26T21:32:13.682558Z","shell.execute_reply":"2024-01-26T21:32:13.687618Z"},"trusted":true},"execution_count":null,"outputs":[]}]}