{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 📋 Table of Contents\n* [File Overview](#files)\n* [Training File](#train)\n* [EEG File Example](#ex_EEG)\n* [Spectrogram File Example](#ex_spec)\n* [Test and Submission File](#sub)","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-23T17:37:30.205725Z","iopub.execute_input":"2024-01-23T17:37:30.206178Z","iopub.status.idle":"2024-01-23T17:37:31.563321Z","shell.execute_reply.started":"2024-01-23T17:37:30.206154Z","shell.execute_reply":"2024-01-23T17:37:31.562466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:31.568022Z","iopub.execute_input":"2024-01-23T17:37:31.568913Z","iopub.status.idle":"2024-01-23T17:37:31.573677Z","shell.execute_reply.started":"2024-01-23T17:37:31.568879Z","shell.execute_reply":"2024-01-23T17:37:31.572437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='files'></a>\n# File Overview","metadata":{}},{"cell_type":"code","source":"!ls -l '../input/hms-harmful-brain-activity-classification'","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:31.575045Z","iopub.execute_input":"2024-01-23T17:37:31.575621Z","iopub.status.idle":"2024-01-23T17:37:31.904949Z","shell.execute_reply.started":"2024-01-23T17:37:31.575549Z","shell.execute_reply":"2024-01-23T17:37:31.903998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l '../input/hms-harmful-brain-activity-classification/example_figures'","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:31.907558Z","iopub.execute_input":"2024-01-23T17:37:31.907901Z","iopub.status.idle":"2024-01-23T17:37:32.221193Z","shell.execute_reply.started":"2024-01-23T17:37:31.907864Z","shell.execute_reply":"2024-01-23T17:37:32.220458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l '../input/hms-harmful-brain-activity-classification/train_spectrograms'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-23T17:37:32.222039Z","iopub.execute_input":"2024-01-23T17:37:32.222267Z","iopub.status.idle":"2024-01-23T17:37:33.608387Z","shell.execute_reply.started":"2024-01-23T17:37:32.222242Z","shell.execute_reply":"2024-01-23T17:37:33.607355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l '../input/hms-harmful-brain-activity-classification/train_eegs'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-23T17:37:33.609968Z","iopub.execute_input":"2024-01-23T17:37:33.610309Z","iopub.status.idle":"2024-01-23T17:37:35.677610Z","shell.execute_reply.started":"2024-01-23T17:37:33.610277Z","shell.execute_reply":"2024-01-23T17:37:35.676668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='train'></a>\n# Training File","metadata":{}},{"cell_type":"code","source":"# training data - load and preview\ndf_train = pd.read_csv('../input/hms-harmful-brain-activity-classification/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:35.679213Z","iopub.execute_input":"2024-01-23T17:37:35.679571Z","iopub.status.idle":"2024-01-23T17:37:35.876745Z","shell.execute_reply.started":"2024-01-23T17:37:35.679537Z","shell.execute_reply":"2024-01-23T17:37:35.875922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# votes columns\nfeatures_vote = ['seizure_vote', 'lpd_vote', 'gpd_vote',\n                 'lrda_vote', 'grda_vote', 'other_vote']","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:35.877561Z","iopub.execute_input":"2024-01-23T17:37:35.877762Z","iopub.status.idle":"2024-01-23T17:37:35.882008Z","shell.execute_reply.started":"2024-01-23T17:37:35.877743Z","shell.execute_reply":"2024-01-23T17:37:35.881276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by eeg_id to get just one row per EEG\ndf_train_unique_votes = df_train.groupby('eeg_id')[features_vote].mean()\n\n# calc sum of votes for each row\ndf_train_unique_votes['vote_sum'] = df_train_unique_votes[features_vote].sum(axis=1)\n\n# normalize votes (=> 100% distribution for each row/EEG)\nfor f in features_vote:\n    df_train_unique_votes[f] = df_train_unique_votes[f] / df_train_unique_votes.vote_sum\n    \ndf_train_unique_votes.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:35.882876Z","iopub.execute_input":"2024-01-23T17:37:35.883207Z","iopub.status.idle":"2024-01-23T17:37:35.985256Z","shell.execute_reply.started":"2024-01-23T17:37:35.883185Z","shell.execute_reply":"2024-01-23T17:37:35.984471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='ex_EEG'></a>\n# EEG File Example","metadata":{}},{"cell_type":"code","source":"# load an EEG file\ndf_eeg = pd.read_parquet('../input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:35.988098Z","iopub.execute_input":"2024-01-23T17:37:35.988332Z","iopub.status.idle":"2024-01-23T17:37:36.157380Z","shell.execute_reply.started":"2024-01-23T17:37:35.988311Z","shell.execute_reply":"2024-01-23T17:37:36.156784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_eeg.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:36.158122Z","iopub.execute_input":"2024-01-23T17:37:36.158404Z","iopub.status.idle":"2024-01-23T17:37:36.177403Z","shell.execute_reply.started":"2024-01-23T17:37:36.158376Z","shell.execute_reply":"2024-01-23T17:37:36.176761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show structure\ndf_eeg.info()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:36.179468Z","iopub.execute_input":"2024-01-23T17:37:36.179815Z","iopub.status.idle":"2024-01-23T17:37:36.195460Z","shell.execute_reply.started":"2024-01-23T17:37:36.179786Z","shell.execute_reply":"2024-01-23T17:37:36.194680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basis stats\ndf_eeg.describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:36.196820Z","iopub.execute_input":"2024-01-23T17:37:36.197118Z","iopub.status.idle":"2024-01-23T17:37:36.254321Z","shell.execute_reply.started":"2024-01-23T17:37:36.197090Z","shell.execute_reply":"2024-01-23T17:37:36.253256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of features\nfeatures_eeg = ['Fp1', 'F3', 'C3', 'P3', 'F7', \n                'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz',\n                'Fp2', 'F4', 'C4', 'P4', 'F8',\n                'T4', 'T6', 'O2', 'EKG']","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:36.255482Z","iopub.execute_input":"2024-01-23T17:37:36.256138Z","iopub.status.idle":"2024-01-23T17:37:36.261098Z","shell.execute_reply.started":"2024-01-23T17:37:36.256103Z","shell.execute_reply":"2024-01-23T17:37:36.260354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all features - time series\nfor f in features_eeg:\n    plt.figure(figsize=(12,3))\n    plt.plot(df_eeg[f], color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:36.261998Z","iopub.execute_input":"2024-01-23T17:37:36.262786Z","iopub.status.idle":"2024-01-23T17:37:41.586952Z","shell.execute_reply.started":"2024-01-23T17:37:36.262750Z","shell.execute_reply":"2024-01-23T17:37:41.586103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all features - distributions\nfor f in features_eeg:\n    plt.figure(figsize=(12,3))\n    plt.hist(df_eeg[f], bins=100,\n             color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:41.587773Z","iopub.execute_input":"2024-01-23T17:37:41.588038Z","iopub.status.idle":"2024-01-23T17:37:47.288820Z","shell.execute_reply.started":"2024-01-23T17:37:41.588017Z","shell.execute_reply":"2024-01-23T17:37:47.287985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation:","metadata":{}},{"cell_type":"code","source":"# correlation matrix\ncor_eeg = df_eeg[features_eeg].corr(method='pearson')\nplt.figure(figsize=(12,8))\nsns.heatmap(cor_eeg, annot=True,\n            fmt='.2f',\n            linecolor='black', linewidths=.5,\n            cmap='RdYlGn', vmin=-1, vmax=+1)\nplt.title('Correlation - EEG example')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:47.290189Z","iopub.execute_input":"2024-01-23T17:37:47.290527Z","iopub.status.idle":"2024-01-23T17:37:48.263110Z","shell.execute_reply.started":"2024-01-23T17:37:47.290498Z","shell.execute_reply":"2024-01-23T17:37:48.260227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='ex_spec'></a>\n# Spectrogram File Example","metadata":{}},{"cell_type":"code","source":"# load spectrogram file\ndf_spec = pd.read_parquet('../input/hms-harmful-brain-activity-classification/train_spectrograms/1000646093.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:48.264311Z","iopub.execute_input":"2024-01-23T17:37:48.264563Z","iopub.status.idle":"2024-01-23T17:37:48.311733Z","shell.execute_reply.started":"2024-01-23T17:37:48.264541Z","shell.execute_reply":"2024-01-23T17:37:48.310977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_spec.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:48.312756Z","iopub.execute_input":"2024-01-23T17:37:48.313041Z","iopub.status.idle":"2024-01-23T17:37:48.579437Z","shell.execute_reply.started":"2024-01-23T17:37:48.313014Z","shell.execute_reply":"2024-01-23T17:37:48.578667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show structure\ndf_spec.info(verbose=True, show_counts=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-23T17:37:48.580315Z","iopub.execute_input":"2024-01-23T17:37:48.580538Z","iopub.status.idle":"2024-01-23T17:37:48.606523Z","shell.execute_reply.started":"2024-01-23T17:37:48.580517Z","shell.execute_reply":"2024-01-23T17:37:48.605591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basis stats\ndf_spec.describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:48.607802Z","iopub.execute_input":"2024-01-23T17:37:48.608387Z","iopub.status.idle":"2024-01-23T17:37:49.498610Z","shell.execute_reply.started":"2024-01-23T17:37:48.608363Z","shell.execute_reply":"2024-01-23T17:37:49.497868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export to file\ndf_spec.to_csv('spec_example.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:40:06.166039Z","iopub.execute_input":"2024-01-23T17:40:06.166831Z","iopub.status.idle":"2024-01-23T17:40:06.225981Z","shell.execute_reply.started":"2024-01-23T17:40:06.166799Z","shell.execute_reply":"2024-01-23T17:40:06.225138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Some plots:","metadata":{}},{"cell_type":"code","source":"# plotting function given a specific frequency\ndef plot_spec(i_frequency):\n    feature_1 = 'LL_' + i_frequency\n    feature_2 = 'RL_' + i_frequency\n    feature_3 = 'RP_' + i_frequency\n    feature_4 = 'LP_' + i_frequency\n    plt.figure(figsize=(10,3))\n    plt.scatter(df_spec.time, df_spec[feature_1], s=5, label='LL')\n    plt.scatter(df_spec.time, df_spec[feature_2], s=5, label='RL')\n    plt.scatter(df_spec.time, df_spec[feature_3], s=5, label='RP')\n    plt.scatter(df_spec.time, df_spec[feature_4], s=5, label='LP')\n    plt.legend(loc='upper right')\n    plt.title('Frequency=' + i_frequency)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:49.499488Z","iopub.execute_input":"2024-01-23T17:37:49.500232Z","iopub.status.idle":"2024-01-23T17:37:49.505452Z","shell.execute_reply.started":"2024-01-23T17:37:49.500209Z","shell.execute_reply":"2024-01-23T17:37:49.504730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot for a few frequencies\nfrequencies = ['0.59', '3.91', '6.45', '10.16', '19.92']\nfor freq in frequencies:\n    plot_spec(freq)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:49.506262Z","iopub.execute_input":"2024-01-23T17:37:49.506507Z","iopub.status.idle":"2024-01-23T17:37:50.792457Z","shell.execute_reply.started":"2024-01-23T17:37:49.506482Z","shell.execute_reply":"2024-01-23T17:37:50.791661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# frequencies\nfreqs = [0.59, 0.78, 0.98, 1.17, 1.37, 1.56, 1.76, 1.95, 2.15, 2.34, 2.54, 2.73, 2.93, 3.13, 3.32, \n         3.52, 3.71, 3.91, 4.1, 4.3, 4.49, 4.69, 4.88, 5.08, 5.27, 5.47, 5.66, 5.86, 6.05, 6.25,\n         6.45, 6.64, 6.84, 7.03, 7.23, 7.42, 7.62, 7.81, 8.01, 8.2, 8.4, 8.59, 8.79, 8.98, 9.18,\n         9.38, 9.57, 9.77, 9.96, 10.16, 10.35, 10.55, 10.74, 10.94, 11.13, 11.33, 11.52, 11.72,\n         11.91, 12.11, 12.3, 12.5, 12.7, 12.89, 13.09, 13.28, 13.48, 13.67, 13.87, 14.06, 14.26,\n         14.45, 14.65, 14.84, 15.04, 15.23, 15.43, 15.63, 15.82, 16.02, 16.21, 16.41, 16.6, 16.8,\n         16.99, 17.19, 17.38, 17.58, 17.77, 17.97, 18.16, 18.36, 18.55, 18.75, 18.95, 19.14, 19.34,\n         19.53, 19.73, 19.92]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:51:32.006866Z","iopub.execute_input":"2024-01-23T17:51:32.007190Z","iopub.status.idle":"2024-01-23T17:51:32.013219Z","shell.execute_reply.started":"2024-01-23T17:51:32.007163Z","shell.execute_reply":"2024-01-23T17:51:32.012564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LL columns for all frequencies\ncols_LL = ['LL_' + str(f) for f in freqs]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:51:43.202303Z","iopub.execute_input":"2024-01-23T17:51:43.202619Z","iopub.status.idle":"2024-01-23T17:51:43.206581Z","shell.execute_reply.started":"2024-01-23T17:51:43.202594Z","shell.execute_reply":"2024-01-23T17:51:43.205746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats\ndf_spec[cols_LL].describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:54:06.119199Z","iopub.execute_input":"2024-01-23T17:54:06.119560Z","iopub.status.idle":"2024-01-23T17:54:06.326225Z","shell.execute_reply.started":"2024-01-23T17:54:06.119533Z","shell.execute_reply":"2024-01-23T17:54:06.325402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all frequencies in one chart\nplt.figure(figsize=(12,5))\nplt.plot(df_spec[cols_LL])\nplt.title('LL Measurements - All Frequencies')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:54:08.430636Z","iopub.execute_input":"2024-01-23T17:54:08.430957Z","iopub.status.idle":"2024-01-23T17:54:08.744072Z","shell.execute_reply.started":"2024-01-23T17:54:08.430914Z","shell.execute_reply":"2024-01-23T17:54:08.743103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all frequencies in one chart - log scale\nplt.figure(figsize=(12,5))\nplt.plot(np.log10(df_spec[cols_LL]))\nplt.title('LL Measurements - All Frequencies / log10 scale')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:54:12.667236Z","iopub.execute_input":"2024-01-23T17:54:12.667611Z","iopub.status.idle":"2024-01-23T17:54:13.109945Z","shell.execute_reply.started":"2024-01-23T17:54:12.667583Z","shell.execute_reply":"2024-01-23T17:54:13.109134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='sub'></a>\n# Test and Submission File","metadata":{}},{"cell_type":"code","source":"# load (dummy) test data\ndf_test = pd.read_csv('../input/hms-harmful-brain-activity-classification/test.csv')\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:51.757096Z","iopub.execute_input":"2024-01-23T17:37:51.757313Z","iopub.status.idle":"2024-01-23T17:37:51.769000Z","shell.execute_reply.started":"2024-01-23T17:37:51.757293Z","shell.execute_reply":"2024-01-23T17:37:51.768079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load submission file\ndf_sub = pd.read_csv('../input/hms-harmful-brain-activity-classification/sample_submission.csv')\ndf_sub","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:51.770357Z","iopub.execute_input":"2024-01-23T17:37:51.770590Z","iopub.status.idle":"2024-01-23T17:37:51.784578Z","shell.execute_reply.started":"2024-01-23T17:37:51.770570Z","shell.execute_reply":"2024-01-23T17:37:51.783688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ","metadata":{}},{"cell_type":"markdown","source":"#### Simply use (grouped) means for a first baseline. See also this notebook: https://www.kaggle.com/code/seshurajup/eda-train-csv","metadata":{}},{"cell_type":"code","source":"# simply use means for a first baseline\nmean_values = df_train_unique_votes[features_vote].mean()\n\ndf_sub.seizure_vote = mean_values['seizure_vote']\ndf_sub.lpd_vote = mean_values['lpd_vote']\ndf_sub.gpd_vote = mean_values['gpd_vote']\ndf_sub.lrda_vote = mean_values['lrda_vote']\ndf_sub.grda_vote = mean_values['grda_vote']\ndf_sub.other_vote = mean_values['other_vote']\n\ndf_sub","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:51.785782Z","iopub.execute_input":"2024-01-23T17:37:51.786049Z","iopub.status.idle":"2024-01-23T17:37:51.803083Z","shell.execute_reply.started":"2024-01-23T17:37:51.786024Z","shell.execute_reply":"2024-01-23T17:37:51.802189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save submission file\ndf_sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T17:37:51.803989Z","iopub.execute_input":"2024-01-23T17:37:51.804195Z","iopub.status.idle":"2024-01-23T17:37:51.810659Z","shell.execute_reply.started":"2024-01-23T17:37:51.804175Z","shell.execute_reply":"2024-01-23T17:37:51.809726Z"},"trusted":true},"execution_count":null,"outputs":[]}]}