{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":1493925,"sourceType":"datasetVersion","datasetId":877193},{"sourceId":8032286,"sourceType":"datasetVersion","datasetId":4734573},{"sourceId":8048333,"sourceType":"datasetVersion","datasetId":4746032},{"sourceId":27040,"sourceType":"modelInstanceVersion","modelInstanceId":22783}],"dockerImageVersionId":30674,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":1482.060675,"end_time":"2024-04-07T06:40:04.851658","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-04-07T06:15:22.790983","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport os, gc\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport sys\nimport pickle\n\nimport tensorflow as tf\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nprint(\"TensorFlow version = \",tf.__version__)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":12.798674,"end_time":"2024-04-07T06:15:38.423348","exception":false,"start_time":"2024-04-07T06:15:25.624674","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:19.061015Z","iopub.execute_input":"2024-04-09T05:48:19.061705Z","iopub.status.idle":"2024-04-09T05:48:32.508861Z","shell.execute_reply.started":"2024-04-09T05:48:19.061670Z","shell.execute_reply":"2024-04-09T05:48:32.507940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class paths:\n OUTPUT_DIR = \"/kaggle/working/\"\n BASE_PATH = \"/kaggle/input/hms-harmful-brain-activity-classification\"\n TRAIN_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\"\n TRAIN_EEGS = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\"\n TRAIN_SPECTOGRAMS = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms\"\n \n TEST_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\n TEST_EEGS = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs\"\n TEST_SPECTOGRAMS = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms\"\n \n SAMPLE_SPECTOGRAM = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/1000086677.parquet\"\n SAMPLE_EEG = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet\"\n \n NPY_EEGS = \"/kaggle/input/hms-eeg-cache/eegs.npy\"\n NPY_SPECTOGRAMS = \"/kaggle/input/hms-spectogram-cache/spectrogram.npy\"","metadata":{"papermill":{"duration":0.018115,"end_time":"2024-04-07T06:15:38.450926","exception":false,"start_time":"2024-04-07T06:15:38.432811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.510932Z","iopub.execute_input":"2024-04-09T05:48:32.511454Z","iopub.status.idle":"2024-04-09T05:48:32.517373Z","shell.execute_reply.started":"2024-04-09T05:48:32.511426Z","shell.execute_reply":"2024-04-09T05:48:32.516332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(paths.TRAIN_CSV)\ndf.head(5)","metadata":{"papermill":{"duration":0.268895,"end_time":"2024-04-07T06:15:38.729107","exception":false,"start_time":"2024-04-07T06:15:38.460212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.518574Z","iopub.execute_input":"2024-04-09T05:48:32.518912Z","iopub.status.idle":"2024-04-09T05:48:32.835715Z","shell.execute_reply.started":"2024-04-09T05:48:32.518879Z","shell.execute_reply":"2024-04-09T05:48:32.834823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_cols = df.columns[-6:]\n\nprint(f\"Train dataframe shape is: {df.shape}\")\nprint(f'Unique EEG: {df[\"eeg_id\"].nunique()}')\nprint(f'Unique Spectrogram: {df[\"spectrogram_id\"].nunique()}')\nprint(f\"Labels: {list(label_cols)}\")","metadata":{"papermill":{"duration":0.025886,"end_time":"2024-04-07T06:15:38.765644","exception":false,"start_time":"2024-04-07T06:15:38.739758","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.837140Z","iopub.execute_input":"2024-04-09T05:48:32.837501Z","iopub.status.idle":"2024-04-09T05:48:32.850989Z","shell.execute_reply.started":"2024-04-09T05:48:32.837469Z","shell.execute_reply":"2024-04-09T05:48:32.849884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --------- Transform subsampels into sampels ---------\n# subsamples ===> samples\n\nVOTES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\n\ntrain = df.groupby(\"eeg_id\").agg({\n \"spectrogram_id\": \"first\",\n \"patient_id\": \"first\",\n}).reset_index()\n\n# Adds together all the votes based on the EEG_ID\ntmp = df.groupby(\"eeg_id\")[VOTES].agg(\"sum\")\nfor v in VOTES:\n train[v] = tmp[v].values\n\ntrain[\"target\"] = train[VOTES].sum(axis=1)\ntrain[VOTES] = train[VOTES].div(train[\"target\"], axis=0)\ntrain[\"target\"] = train[VOTES].idxmax(axis=1)\n\n# Shape and View\nprint(f\"Train Shape: {train.shape}\")\ntrain.head()","metadata":{"papermill":{"duration":0.074244,"end_time":"2024-04-07T06:15:38.849579","exception":false,"start_time":"2024-04-07T06:15:38.775335","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.853261Z","iopub.execute_input":"2024-04-09T05:48:32.853557Z","iopub.status.idle":"2024-04-09T05:48:32.912344Z","shell.execute_reply.started":"2024-04-09T05:48:32.853533Z","shell.execute_reply":"2024-04-09T05:48:32.911413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ_SPEC_FILES = False\n\n# files = os.listdir(paths.TRAIN_SPECTOGRAMS)\n# print(f\"There are {len(files)} spectrogram parquets\")\n\n# if READ_SPEC_FILES:\n# spectrograms = {}\n# for i,f in enumerate(files):\n# if i%100==0: print(i,\", \",end=\"\")\n# tmp = pd.read_parquet(f\"{paths.TRAIN_SPECTOGRAMS}/{f}\")\n# name = int(f.split(\".\")[0])\n# spectrograms[name] = tmp.iloc[:,1:].values\n \n# np.save(\"spectrograms.npy\", spectrograms)\n# READ_SPEC_FILES = False\n# else:\n# spectrograms = np.load(\"/kaggle/input/hms-spectogram-cache/spectrograms.npy\",allow_pickle=True).item()","metadata":{"papermill":{"duration":0.018152,"end_time":"2024-04-07T06:15:38.878968","exception":false,"start_time":"2024-04-07T06:15:38.860816","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.913433Z","iopub.execute_input":"2024-04-09T05:48:32.913718Z","iopub.status.idle":"2024-04-09T05:48:32.918287Z","shell.execute_reply.started":"2024-04-09T05:48:32.913695Z","shell.execute_reply":"2024-04-09T05:48:32.917410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ_EEG_FILES = False\n\n# files = os.listdir(paths.TRAIN_EEGS)\n# print(f\"There are {len(files)} eeg parquets\")\n\n# if READ_EEG_FILES:\n# eegs = {}\n# for i,f in enumerate(files):\n# if i%100==0: print(i,\", \",end=\"\")\n# tmp = pd.read_parquet(f\"{paths.TRAIN_EEGS}/{f}\")\n# name = int(f.split(\".\")[0])\n# eegs[name] = tmp.iloc[:,1:].values\n \n# np.save(\"eegs.npy\", eegs)\n# READ_EEG_FILES = False\n# else:\n# eegs = np.load(\"/kaggle/input/hms-eeg-cache/eegs.npy\",allow_pickle=True).item()","metadata":{"papermill":{"duration":0.017649,"end_time":"2024-04-07T06:15:38.906847","exception":false,"start_time":"2024-04-07T06:15:38.889198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.919363Z","iopub.execute_input":"2024-04-09T05:48:32.919701Z","iopub.status.idle":"2024-04-09T05:48:32.928192Z","shell.execute_reply.started":"2024-04-09T05:48:32.919674Z","shell.execute_reply":"2024-04-09T05:48:32.927322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"READ = True\n# Appreciate the smaller mem usage of .npy format \nif READ:\n spectrograms = np.load(paths.NPY_SPECTOGRAMS, allow_pickle=True).item()\n example_spectrogram = train[\"spectrogram_id\"].iloc[0]\n\n print(f\"Size of Spectogram Dictionary: {len(spectrograms)}\")\n print(f\"Shape of Spectogram Item: {spectrograms[example_spectrogram].shape}\")\nelse:\n spectrograms = {}","metadata":{"papermill":{"duration":54.327218,"end_time":"2024-04-07T06:16:33.244159","exception":false,"start_time":"2024-04-07T06:15:38.916941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:48:32.929197Z","iopub.execute_input":"2024-04-09T05:48:32.929536Z","iopub.status.idle":"2024-04-09T05:49:57.479563Z","shell.execute_reply.started":"2024-04-09T05:48:32.929510Z","shell.execute_reply":"2024-04-09T05:49:57.478643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"READ = True\n# Appreciate the smaller mem usage of .npy format \nif READ:\n eegs = np.load(paths.NPY_EEGS, allow_pickle=True).item()\n \n example_eeg = train[\"eeg_id\"].iloc[0]\n\n print(f\"Size of EEG Dictionary: {len(eegs)}\")\n print(f\"Shape of EEG Item: {eegs[example_eeg].shape}\")\nelse:\n eegs = {}","metadata":{"papermill":{"duration":66.682821,"end_time":"2024-04-07T06:17:39.936947","exception":false,"start_time":"2024-04-07T06:16:33.254126","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:49:57.480973Z","iopub.execute_input":"2024-04-09T05:49:57.481259Z","iopub.status.idle":"2024-04-09T05:51:48.590958Z","shell.execute_reply.started":"2024-04-09T05:49:57.481233Z","shell.execute_reply":"2024-04-09T05:51:48.589713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def plot_spectrogram(spectrogram_path: str):\n# \"\"\"\n# Source: https://www.kaggle.com/code/mvvppp/hms-eda-and-domain-journey\n# Visualize spectogram recordings from a parquet file.\n# :param spectrogram_path: path to the spectogram parquet.\n# \"\"\"\n# sample_spect = pd.read_parquet(spectrogram_path)\n \n# split_spect = {\n# \"LL\": sample_spect.filter(regex=\"^LL\", axis=1),\n# \"RL\": sample_spect.filter(regex=\"^RL\", axis=1),\n# \"RP\": sample_spect.filter(regex=\"^RP\", axis=1),\n# \"LP\": sample_spect.filter(regex=\"^LP\", axis=1),\n# }\n \n# fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 12))\n# axes = axes.flatten()\n# label_interval = 5\n# for i, split_name in enumerate(split_spect.keys()):\n# ax = axes[i]\n# img = ax.imshow(np.log(split_spect[split_name]).T, cmap=\"viridis\", aspect=\"auto\", origin=\"lower\")\n# cbar = fig.colorbar(img, ax=ax)\n# cbar.set_label(\"Log(Value)\")\n# ax.set_title(split_name)\n# ax.set_ylabel(\"Frequency (Hz)\")\n# ax.set_xlabel(\"Time\")\n\n# ax.set_yticks(np.arange(len(split_spect[split_name].columns)))\n# ax.set_yticklabels([column_name[3:] for column_name in split_spect[split_name].columns])\n# frequencies = [column_name[3:] for column_name in split_spect[split_name].columns]\n# ax.set_yticks(np.arange(0, len(split_spect[split_name].columns), label_interval))\n# ax.set_yticklabels(frequencies[::label_interval])\n# plt.tight_layout()\n# plt.show()","metadata":{"papermill":{"duration":0.019255,"end_time":"2024-04-07T06:17:39.966459","exception":false,"start_time":"2024-04-07T06:17:39.947204","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.593369Z","iopub.execute_input":"2024-04-09T05:51:48.594365Z","iopub.status.idle":"2024-04-09T05:51:48.600300Z","shell.execute_reply.started":"2024-04-09T05:51:48.594327Z","shell.execute_reply":"2024-04-09T05:51:48.599200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # READ ALL SPECTROGRAMS\n\n# files = os.listdir(paths.TRAIN_SPECTOGRAMS)\n# print(f\"There are {len(files)} spectrogram parquets\")","metadata":{"papermill":{"duration":0.017154,"end_time":"2024-04-07T06:17:39.993806","exception":false,"start_time":"2024-04-07T06:17:39.976652","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.601976Z","iopub.execute_input":"2024-04-09T05:51:48.602490Z","iopub.status.idle":"2024-04-09T05:51:48.616888Z","shell.execute_reply.started":"2024-04-09T05:51:48.602457Z","shell.execute_reply":"2024-04-09T05:51:48.615936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# se = pd.read_parquet(paths.SAMPLE_SPECTOGRAM)\n# se","metadata":{"papermill":{"duration":0.01762,"end_time":"2024-04-07T06:17:40.021860","exception":false,"start_time":"2024-04-07T06:17:40.004240","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.618147Z","iopub.execute_input":"2024-04-09T05:51:48.618915Z","iopub.status.idle":"2024-04-09T05:51:48.626127Z","shell.execute_reply.started":"2024-04-09T05:51:48.618883Z","shell.execute_reply":"2024-04-09T05:51:48.625090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # The resulting all_spectograms dictionary contains spectogram_id as keys (int keys) and the values are\n# # the spectogram sequences (as 2-dimensional np.array) of shape (timesteps, 400).\n\n# # Each spectogram is a parquet file. This parquet, when converted to a pandas dataframe,\n# # results in a dataframe of shape (time_steps, 401). First column is the time column and the remaining 400 columns are the recordings.\n# # There are 400 columns because there are, respectively, 100 rows associated to the 4 recording regions of the EEG electrodes: LL, RL, LP, RP.\n# # Column names also include the frequency in heartz.\n\n# se = pd.read_parquet(paths.SAMPLE_SPECTOGRAM)\n# se","metadata":{"papermill":{"duration":0.017714,"end_time":"2024-04-07T06:17:40.049676","exception":false,"start_time":"2024-04-07T06:17:40.031962","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.627600Z","iopub.execute_input":"2024-04-09T05:51:48.627910Z","iopub.status.idle":"2024-04-09T05:51:48.637229Z","shell.execute_reply.started":"2024-04-09T05:51:48.627879Z","shell.execute_reply":"2024-04-09T05:51:48.636225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx = np.random.randint(0, len(files))\n# spectrogram_path = f\"{paths.TRAIN_SPECTOGRAMS}/\" + files[idx]\n# plot_spectrogram(spectrogram_path)","metadata":{"papermill":{"duration":0.017156,"end_time":"2024-04-07T06:17:40.077039","exception":false,"start_time":"2024-04-07T06:17:40.059883","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.642085Z","iopub.execute_input":"2024-04-09T05:51:48.642955Z","iopub.status.idle":"2024-04-09T05:51:48.648332Z","shell.execute_reply.started":"2024-04-09T05:51:48.642897Z","shell.execute_reply":"2024-04-09T05:51:48.647459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # READ ALL EEGS\n\n# files = os.listdir(paths.TRAIN_EEGS)\n# print(f\"There are {len(files)} eeg parquets\")","metadata":{"papermill":{"duration":0.017285,"end_time":"2024-04-07T06:17:40.104456","exception":false,"start_time":"2024-04-07T06:17:40.087171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.649936Z","iopub.execute_input":"2024-04-09T05:51:48.650258Z","iopub.status.idle":"2024-04-09T05:51:48.657213Z","shell.execute_reply.started":"2024-04-09T05:51:48.650228Z","shell.execute_reply":"2024-04-09T05:51:48.656326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# se = pd.read_parquet(paths.SAMPLE_EEG)\n# se","metadata":{"papermill":{"duration":0.017248,"end_time":"2024-04-07T06:17:40.131749","exception":false,"start_time":"2024-04-07T06:17:40.114501","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.658506Z","iopub.execute_input":"2024-04-09T05:51:48.658828Z","iopub.status.idle":"2024-04-09T05:51:48.665994Z","shell.execute_reply.started":"2024-04-09T05:51:48.658800Z","shell.execute_reply":"2024-04-09T05:51:48.665039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots(20, figsize=(10, 100))\n\n# # Generate a line plot for each column in the DataFrame\n# for i, column in enumerate(se.columns):\n# ax[i].plot(se.index, se[column], label=column)\n# ax[i].grid(True)\n# ax[i].set_title(str(column))\n\n# # plt.legend()\n# # plt.title(\"Simulated Data Line Chart\")\n# # plt.xlabel(\"Index\")\n# # plt.ylabel(\"Values\")\n# # plt.grid(True)\n# plt.show()","metadata":{"papermill":{"duration":0.017395,"end_time":"2024-04-07T06:17:40.159228","exception":false,"start_time":"2024-04-07T06:17:40.141833","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.667203Z","iopub.execute_input":"2024-04-09T05:51:48.668022Z","iopub.status.idle":"2024-04-09T05:51:48.675598Z","shell.execute_reply.started":"2024-04-09T05:51:48.667980Z","shell.execute_reply":"2024-04-09T05:51:48.674687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(train.patient_id.unique())","metadata":{"papermill":{"duration":0.016956,"end_time":"2024-04-07T06:17:40.187710","exception":false,"start_time":"2024-04-07T06:17:40.170754","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.676724Z","iopub.execute_input":"2024-04-09T05:51:48.677696Z","iopub.status.idle":"2024-04-09T05:51:48.684301Z","shell.execute_reply.started":"2024-04-09T05:51:48.677669Z","shell.execute_reply":"2024-04-09T05:51:48.683454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(train.patient_id)","metadata":{"papermill":{"duration":0.017039,"end_time":"2024-04-07T06:17:40.214808","exception":false,"start_time":"2024-04-07T06:17:40.197769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.685419Z","iopub.execute_input":"2024-04-09T05:51:48.686375Z","iopub.status.idle":"2024-04-09T05:51:48.693773Z","shell.execute_reply.started":"2024-04-09T05:51:48.686343Z","shell.execute_reply":"2024-04-09T05:51:48.692869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport keras\n\nclass DataGenerator(keras.utils.Sequence):\n \"Generates data for Keras\"\n def __init__(self,\n data,\n state=\"train\", \n batch_size=32,\n n_classes=6,\n shuffle=True, \n specs=spectrograms,\n eegs=eegs):\n \"Initialization\"\n self.data = data\n self.state = state\n self.batch_size = batch_size\n self.n_classes = n_classes\n self.specs = specs\n self.eegs = eegs\n self.shuffle = shuffle\n self.on_epoch_end()\n\n def __len__(self):\n \"Denotes the number of batches per epoch\"\n return int(np.floor(len(self.data) / self.batch_size))\n \n def __getitem__(self, index):\n \"Generate one batch of data\"\n # Generate indexes of the batch\n indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n # Generate data\n X, y = self.__data_generation(indexes)\n\n return X, y\n\n def on_epoch_end(self):\n \"Updates indexes after each epoch\"\n self.indexes = np.arange(len(self.data))\n if self.shuffle == True:\n np.random.shuffle(self.indexes)\n\n def __data_generation(self, indexes):\n \"Generates data containing batch_size samples\" # X : (n_samples, *dim, n_channels)\n # Initialization\n X = np.zeros((len(indexes),128,256,4),dtype=\"float32\")\n y = np.zeros((len(indexes),self.n_classes),dtype=\"float32\")\n img = np.ones((128,256),dtype=\"float32\")\n \n for n, i in enumerate(indexes):\n \n # date row\n cur = self.data.iloc[i]\n \n # images of spec and eeg\n spec = self.specs[cur.spectrogram_id] # Not in use as of now\n eeg = self.eegs[cur.eeg_id]\n\n # add the data on\n X[n] = eeg\n \n if self.state == \"train\":\n y[n,] = cur[VOTES]\n \n return X, y","metadata":{"papermill":{"duration":0.025792,"end_time":"2024-04-07T06:17:40.250555","exception":false,"start_time":"2024-04-07T06:17:40.224763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.697292Z","iopub.execute_input":"2024-04-09T05:51:48.697575Z","iopub.status.idle":"2024-04-09T05:51:48.710297Z","shell.execute_reply.started":"2024-04-09T05:51:48.697552Z","shell.execute_reply":"2024-04-09T05:51:48.709294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the Data Shape and for Testing\nif READ:\n gen = DataGenerator(train,\n batch_size=6)\n\n# gen.__getitem__(0)","metadata":{"papermill":{"duration":0.020422,"end_time":"2024-04-07T06:17:40.281185","exception":false,"start_time":"2024-04-07T06:17:40.260763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.711479Z","iopub.execute_input":"2024-04-09T05:51:48.711753Z","iopub.status.idle":"2024-04-09T05:51:48.725738Z","shell.execute_reply.started":"2024-04-09T05:51:48.711731Z","shell.execute_reply":"2024-04-09T05:51:48.724737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install kaggle-metrics","metadata":{"papermill":{"duration":0.017241,"end_time":"2024-04-07T06:17:40.308395","exception":false,"start_time":"2024-04-07T06:17:40.291154","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.726857Z","iopub.execute_input":"2024-04-09T05:51:48.727587Z","iopub.status.idle":"2024-04-09T05:51:48.738581Z","shell.execute_reply.started":"2024-04-09T05:51:48.727560Z","shell.execute_reply":"2024-04-09T05:51:48.737629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\n# building model\ndef build_cnn():\n inp = tf.keras.Input(shape=(128, 256, 4))\n \n # EfficientNetB0 Base Model\n \n base_model = tf.keras.applications.efficientnet.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n base_model.load_weights('/kaggle/input/keras-pretrained-models/EfficientNetB0_NoTop_ImageNet.h5')\n \n \n x = [inp[:,:,:,i:i+1] for i in range(4)]\n x = tf.keras.layers.Concatenate(axis=1)(x)\n \n \n # Model and Additional layera\n x = base_model(x)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n\n model = tf.keras.Model(inputs=inp, outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate=1e-3)\n loss = tf.keras.losses.KLDivergence()\n \n model.compile(loss=loss, optimizer=opt, metrics=['acc'])\n \n model.summary()\n \n return model","metadata":{"papermill":{"duration":0.024391,"end_time":"2024-04-07T06:17:40.342836","exception":false,"start_time":"2024-04-07T06:17:40.318445","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:51:48.739825Z","iopub.execute_input":"2024-04-09T05:51:48.740114Z","iopub.status.idle":"2024-04-09T05:54:05.486043Z","shell.execute_reply.started":"2024-04-09T05:51:48.740090Z","shell.execute_reply":"2024-04-09T05:54:05.484728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = tf.distribute.MirroredStrategy()\n\nTRAIN = True\n\nif TRAIN:\n gen = DataGenerator(data=train, \n batch_size=32, \n n_classes=6,\n shuffle=True)\n\n with strategy.scope():\n model = build_cnn() \n model._name = \"EfficientNet\"\n \n checkpoint_callback = ModelCheckpoint(f'Model/EfficientNet_best.keras', \n monitor='acc', \n verbose=1, \n save_best_only=True, \n mode='max')\n\n model.fit(gen,\n verbose=1, \n epochs=4,\n callbacks=[checkpoint_callback])","metadata":{"papermill":{"duration":1318.063905,"end_time":"2024-04-07T06:39:38.416840","exception":false,"start_time":"2024-04-07T06:17:40.352935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:54:05.487069Z","iopub.status.idle":"2024-04-09T05:54:05.487484Z","shell.execute_reply.started":"2024-04-09T05:54:05.487257Z","shell.execute_reply":"2024-04-09T05:54:05.487295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(paths.TEST_CSV)\n\ntest_df = test_df.groupby(\"eeg_id\").agg({\n \"spectrogram_id\": \"first\",\n \"patient_id\": \"first\",\n}).reset_index()\n\n# Shape and View\nprint(f\"Train Shape: {test_df.shape}\")\ntest_df.head()","metadata":{"papermill":{"duration":0.238008,"end_time":"2024-04-07T06:39:38.837278","exception":false,"start_time":"2024-04-07T06:39:38.599270","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:54:05.489061Z","iopub.status.idle":"2024-04-09T05:54:05.489409Z","shell.execute_reply.started":"2024-04-09T05:54:05.489220Z","shell.execute_reply":"2024-04-09T05:54:05.489234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \nNAMES = [\"LL\",\"LP\",\"RP\",\"RR\"]\n\nFEATS = [[\"Fp1\",\"F7\",\"T3\",\"T5\",\"O1\"],\n [\"Fp1\",\"F3\",\"C3\",\"P3\",\"O1\"],\n [\"Fp2\",\"F8\",\"T4\",\"T6\",\"O2\"],\n [\"Fp2\",\"F4\",\"C4\",\"P4\",\"O2\"]]\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n \n # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n eeg = pd.read_parquet(parquet_path)\n middle = (len(eeg)-10_000)//2\n eeg = eeg.iloc[middle:middle+10_000]\n \n # VARIABLE TO HOLD SPECTROGRAM\n img = np.zeros((128,256,4),dtype=\"float32\")\n \n if display: plt.figure(figsize=(10,7))\n signals = []\n for k in range(4):\n COLS = FEATS[k]\n \n for kk in range(4):\n \n # COMPUTE PAIR DIFFERENCES\n x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n # FILL NANS\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n\n # DENOISE\n if USE_WAVELET:\n x = denoise(x, wavelet=USE_WAVELET)\n signals.append(x)\n\n # RAW SPECTROGRAM\n mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n # LOG TRANSFORM\n width = (mel_spec.shape[1]//32)*32\n mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n # STANDARDIZE TO -1 TO 1\n mel_spec_db = (mel_spec_db+40)/40 \n img[:,:,k] += mel_spec_db\n \n # AVERAGE THE 4 MONTAGE DIFFERENCES\n img[:,:,k] /= 4.0\n \n if display:\n plt.subplot(2,2,k+1)\n plt.imshow(img[:,:,k],aspect=\"auto\",origin=\"lower\")\n plt.title(f\"EEG {eeg_id} - Spectrogram {NAMES[k]}\")\n \n if display: \n plt.show()\n plt.figure(figsize=(10,5))\n offset = 0\n for k in range(4):\n if k>0: offset -= signals[3-k].min()\n plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n offset += signals[3-k].max()\n plt.legend()\n plt.title(f\"EEG {eeg_id} Signals\")\n plt.show()\n print(); print(\"#\"*25); print()\n \n return img","metadata":{"papermill":{"duration":0.354767,"end_time":"2024-04-07T06:39:39.390277","exception":false,"start_time":"2024-04-07T06:39:39.035510","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:54:05.490749Z","iopub.status.idle":"2024-04-09T05:54:05.491084Z","shell.execute_reply.started":"2024-04-09T05:54:05.490921Z","shell.execute_reply":"2024-04-09T05:54:05.490935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test_df.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n \n # CREATE SPECTROGRAM FROM EEG PARQUET\n img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i