{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7249608,"sourceType":"datasetVersion","datasetId":4200207},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7403069,"sourceType":"datasetVersion","datasetId":4304949},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":7597816,"sourceType":"datasetVersion","datasetId":4422672},{"sourceId":7606898,"sourceType":"datasetVersion","datasetId":4395866},{"sourceId":7631210,"sourceType":"datasetVersion","datasetId":4446472},{"sourceId":7751992,"sourceType":"datasetVersion","datasetId":4456721,"isSourceIdPinned":false}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":270.012179,"end_time":"2024-01-14T22:56:02.916427","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-14T22:51:32.904248","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# <span id='00' style='color:#2f6faa'> HMS Harm Brain Activity Classification </span>\n## <span id='00' style='color:#2f6faa'>  INFER TF 2.15 Keras 2.15 on GPU P100</span>\n\n### Train on TPU\n- This [notebook](https://www.kaggle.com/mcwema/hms-hbac-train-tf2-15-keras3-0-2-tpu/edit)\n\n### Credit \n - This work is based on this [notebook](https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43)\n\n\n\n\n## <a id='0'> Content </a>\n\n1. [Install](#Install)\n2. [Import](#Import)\n3. [GPU Setting](#Device)\n4. [Config](#Config)\n5. [Load Train Config](#TrainConfig)\n6. [Data](#Data)\n    - [Create Non Overlaping eeg_id](#NoOverlap)\n    - [Read Train Spectrograms](#ReadSpectr)\n    - [Read EEG Spectrograms](#ReadEegSpectr)\n    - [Data Loader](#Loader)\n    - [Visualize](#Visual)\n6. [Learning Schedule](#Learn)\n6. [Build Model](#Model)\n6. [Evaluate](#Eval)\n5. [Save Config](#Save)\n6. [Score](#Score)\n6. [Infer ](#Infer)","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport yaml\nimport datetime\n\nimport tensorflow as tf\nimport keras\n\ntf.__version__, keras.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reproducibility","metadata":{}},{"cell_type":"code","source":"keras.utils.set_random_seed(2530)","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:29:39.718406Z","iopub.execute_input":"2024-02-24T18:29:39.719063Z","iopub.status.idle":"2024-02-24T18:29:39.723897Z","shell.execute_reply.started":"2024-02-24T18:29:39.719028Z","shell.execute_reply":"2024-02-24T18:29:39.7229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  Setup Device\n## <a id ='Device' href='#0'> Setup Device </a>","metadata":{}},{"cell_type":"code","source":"ON_TPU = False\nTPUS =  tf.config.list_logical_devices('TPU')\nif TPUS:\n    cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='local')\n    tf.config.experimental_connect_to_cluster(cluster_resolver)\n    tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n    strategy = tf.distribute.TPUStrategy(cluster_resolver)\n    DEVICE = \"TPU\"\n    REPLICAS = len(TPUS)\n    ON_TPU = True\n    print(\"ON GPU: Replicas:\", REPLICAS, TPUS)\nelse:\n    GPUS =  tf.config.list_logical_devices('GPU')\n    if GPUS:\n        ON_GPU = True\n        DEVICE = \"GPU\"\n        REPLICAS = len(GPUS)\n        \n        if len(GPUS) == 1: \n            strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n\n        else: \n            os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n            strategy = tf.distribute.MirroredStrategy()\n        print(\"ON GPU: Replicas:\", REPLICAS, GPUS)\n    \n    else:\n        CPUS =  tf.config.list_logical_devices('CPU')\n        REPLICAS = len(CPUS)\n        strategy = tf.distribute.MirroredStrategy()\n        print(\"ON CPU. All devices: \", CPUS, \"Replicas:\", REPLICAS)","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:47:27.299374Z","iopub.execute_input":"2024-02-24T18:47:27.300223Z","iopub.status.idle":"2024-02-24T18:47:27.311263Z","shell.execute_reply.started":"2024-02-24T18:47:27.300186Z","shell.execute_reply":"2024-02-24T18:47:27.310326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Install \n## <a id ='Install' href='#0'> Install </a>","metadata":{}},{"cell_type":"code","source":"if ON_TPU:\n    !pip  install -q pyarrow # for parquet\n    !pip  install -q fastparquet # parquet\n    !pip  install -q librosa\n    !pip  install -q PyWavelets # pywt\n    !pip  install -q albumentations\n    REPLICAS = len(TPUS)\n    REPLICAS ","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:47:40.849423Z","iopub.execute_input":"2024-02-24T18:47:40.850141Z","iopub.status.idle":"2024-02-24T18:47:40.858383Z","shell.execute_reply.started":"2024-02-24T18:47:40.850112Z","shell.execute_reply":"2024-02-24T18:47:40.857515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='Config' href='#0'> Config </a>","metadata":{}},{"cell_type":"code","source":"RUNID     = 5\nRUN_DATE  = '{:%Y-%m-%d %H:%M}'.format(datetime.datetime.now())\nDEBUG     = False\nWEIGHTED  = False\nCVSCORES  = True\nRUN_DATE","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:47:43.735267Z","iopub.execute_input":"2024-02-24T18:47:43.735899Z","iopub.status.idle":"2024-02-24T18:47:43.742771Z","shell.execute_reply.started":"2024-02-24T18:47:43.735867Z","shell.execute_reply":"2024-02-24T18:47:43.741751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport albumentations as albu\n\nfrom pathlib import Path\nfrom sklearn.model_selection import KFold, GroupKFold\n\nfrom yaml import CLoader as Loader, CDumper as Dumper\n\nimport math\nimport sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nsys.path.append('/kaggle/input/my-kld-scripts')\nfrom kaggle_kl_div import score\n\n\nSAVE_PATH = Path(\"/kaggle/working/Results/\",  exist_ok=True)\n\n###########  Adjust to model input dir  ############\nLOAD_MODELS_FROM  = '/kaggle/input/tfk-2-15-3-0-2-runs/models/' \n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True\n\n!mkdir -p \"/kaggle/working/Results/\"","metadata":{"papermill":{"duration":14.80928,"end_time":"2024-01-14T22:51:51.64702","exception":false,"start_time":"2024-01-14T22:51:36.83774","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-24T18:47:48.490937Z","iopub.execute_input":"2024-02-24T18:47:48.491625Z","iopub.status.idle":"2024-02-24T18:47:49.505166Z","shell.execute_reply.started":"2024-02-24T18:47:48.491588Z","shell.execute_reply":"2024-02-24T18:47:49.503933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='TrainConfig' href='#0'> Load Train Config </a>","metadata":{}},{"cell_type":"code","source":"try:\n    Config_File = [f for f in os.listdir(f'{LOAD_MODELS_FROM}') if 'Config.yaml' in f][0]\n    History_Files = [f for f in os.listdir(f'{LOAD_MODELS_FROM}') if 'History.yaml' in f]\n    assert(os.path.exists(f\"{LOAD_MODELS_FROM}{Config_File}\"))\n    TrainConfig = yaml.load(open(f\"{LOAD_MODELS_FROM}{Config_File}\", \"r\"),  Loader=yaml.FullLoader)\n    \n    History_Results = [yaml.load(open(f\"{LOAD_MODELS_FROM}{file}\", \"r\"),  Loader=yaml.FullLoader) for file in History_Files]\n\nexcept:\n    print(\"\\n\",'#'*5,f\"\\t Config File not found \")\n\n\nPARAMS = [\"MODEL\", \"BASE\", \"TRAINID\", \"RUN_DATE\", \"TUNE_TRAIN\", \"FINE_TUNE\", \"TRAIN_BATCH\", \"PATIENCE\"]\n\ndef show_params():\n    print( \"Key Params: \", [f\"{key} = {TrainConfig[key]}\" for key in PARAMS])\n\nshow_params()","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:47:53.727008Z","iopub.execute_input":"2024-02-24T18:47:53.727711Z","iopub.status.idle":"2024-02-24T18:47:53.798837Z","shell.execute_reply.started":"2024-02-24T18:47:53.727677Z","shell.execute_reply":"2024-02-24T18:47:53.797916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_hist(hist, key):\n    plt.plot(hist[key])\n    plt.plot(hist[f\"val_{key}\"])\n    plt.title(f\"model {key}\")\n    plt.ylabel(f\"{key}\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()\n\ndef PloTrainHistory(i):\n    hist = History_Results[i]\n    fig, axs = plt.subplots(1, 3, figsize=(10, 2), layout='constrained')\n    for k, key in enumerate([\"loss\", \"binary_accuracy\",  \"categorical_accuracy\"]):\n        axs[k].plot(hist[key])\n        axs[k].plot(hist[f\"val_{key}\"]) \n        if i == 0:\n            axs[k].set_title(f\"{key}\")\n        if k == 0:\n            axs[k].set_ylabel(f\"Fold {i}\")\n        if i == len(History_Results) - 1:\n            axs[k].set_xlabel(\"epoch\")\n        axs[k].legend([\"train\", \"validation\"], loc=\"upper left\")\n\n    #fig.suptitle(f'Fold {i+1}')\n    fig.get_layout_engine().set(w_pad=3 / 62, h_pad=3 / 62, hspace=0.2,\n                            wspace=0.2)\n\nfor i in range(len(History_Results)):\n    PloTrainHistory(i)","metadata":{"execution":{"iopub.status.busy":"2024-02-24T18:47:59.338122Z","iopub.execute_input":"2024-02-24T18:47:59.338832Z","iopub.status.idle":"2024-02-24T18:47:59.344338Z","shell.execute_reply.started":"2024-02-24T18:47:59.338801Z","shell.execute_reply":"2024-02-24T18:47:59.343366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    #print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"papermill":{"duration":0.016556,"end_time":"2024-01-14T22:51:51.671783","exception":false,"start_time":"2024-01-14T22:51:51.655227","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:45:28.048783Z","iopub.execute_input":"2024-02-21T15:45:28.0492Z","iopub.status.idle":"2024-02-21T15:45:28.056508Z","shell.execute_reply.started":"2024-02-21T15:45:28.049144Z","shell.execute_reply":"2024-02-21T15:45:28.055302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='Data' href='#0'> Load Data </a>","metadata":{"papermill":{"duration":0.007846,"end_time":"2024-01-14T22:51:51.688268","exception":false,"start_time":"2024-01-14T22:51:51.680422","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndisplay(df.head())","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.288611,"end_time":"2024-01-14T22:51:51.984993","exception":false,"start_time":"2024-01-14T22:51:51.696382","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:45:28.057899Z","iopub.execute_input":"2024-02-21T15:45:28.058276Z","iopub.status.idle":"2024-02-21T15:45:28.348533Z","shell.execute_reply.started":"2024-02-21T15:45:28.058247Z","shell.execute_reply":"2024-02-21T15:45:28.34752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='NoOverlap' href='#0'> Create  Non Overlapping Id </a>\nThe competition data description says that test data does not have multiple crops from the same `eeg_id`. Therefore we will train and validate using only 1 crop per `eeg_id`. There is a discussion about this [here][1].\n\n[1]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/467021","metadata":{"papermill":{"duration":0.009407,"end_time":"2024-01-14T22:51:52.004075","exception":false,"start_time":"2024-01-14T22:51:51.994668","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if CVSCORES:\n    train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n        {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\n    train.columns = ['spec_id','min']\n\n    tmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n        {'spectrogram_label_offset_seconds':'max'})\n    train['max'] = tmp\n\n    tmp = df.groupby('eeg_id')[['patient_id']].agg('first')\n    train['patient_id'] = tmp\n\n    tmp = df.groupby('eeg_id')[TARGETS].agg('sum')\n    for t in TARGETS:\n        train[t] = tmp[t].values\n\n    y_data = train[TARGETS].values\n    y_data = y_data / y_data.sum(axis=1,keepdims=True)\n    train[TARGETS] = y_data\n\n    tmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\n    train['target'] = tmp\n\n    train = train.reset_index()\n    print('Train non-overlapp eeg_id shape:', train.shape )\n    display(train.head())","metadata":{"papermill":{"duration":0.111621,"end_time":"2024-01-14T22:51:52.125134","exception":false,"start_time":"2024-01-14T22:51:52.013513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:45:28.349743Z","iopub.execute_input":"2024-02-21T15:45:28.350027Z","iopub.status.idle":"2024-02-21T15:45:28.444093Z","shell.execute_reply.started":"2024-02-21T15:45:28.350003Z","shell.execute_reply":"2024-02-21T15:45:28.443173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    train = train.sample(frac=0.250, replace=True, random_state=1)\n    print(f'Using DEBUG Config. Train non-overlapp eeg_id shape: {train.shape}' )","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:45:28.44572Z","iopub.execute_input":"2024-02-21T15:45:28.446038Z","iopub.status.idle":"2024-02-21T15:45:28.450931Z","shell.execute_reply.started":"2024-02-21T15:45:28.446011Z","shell.execute_reply":"2024-02-21T15:45:28.449965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CVSCORES:\n    targets = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n    for target in targets:\n\n        Target_df = train[train[target] > 0]\n        print(target, Target_df.shape)\n        display(Target_df.head(), Target_df[Target_df['other_vote'] > 0].shape )","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:45:28.452373Z","iopub.execute_input":"2024-02-21T15:45:28.452751Z","iopub.status.idle":"2024-02-21T15:45:28.57335Z","shell.execute_reply.started":"2024-02-21T15:45:28.452717Z","shell.execute_reply":"2024-02-21T15:45:28.572214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='ReadSpectr' href='#0'> Read Spectrograms </a>\n\nFirst we need to read in all 11k train spectrogram files. Reading thousands of files takes 11 minutes with Pandas. Instead, we can read 1 file from [Kaggle dataset here][1] which contains all the 11k spectrograms in less than 1 minute! To use my Kaggle dataset, set variable `READ_SPEC_FILES = False`. Thank you for upvoting my helpful [dataset][1] :-)\n\n[1]: https://www.kaggle.com/datasets/cdeotte/brain-spectrograms","metadata":{"papermill":{"duration":0.00881,"end_time":"2024-01-14T22:51:52.142747","exception":false,"start_time":"2024-01-14T22:51:52.133937","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n\nREAD_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"papermill":{"duration":55.16894,"end_time":"2024-01-14T22:52:47.320438","exception":false,"start_time":"2024-01-14T22:51:52.151498","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:45:28.574779Z","iopub.execute_input":"2024-02-21T15:45:28.575199Z","iopub.status.idle":"2024-02-21T15:46:38.787172Z","shell.execute_reply.started":"2024-02-21T15:45:28.575144Z","shell.execute_reply":"2024-02-21T15:46:38.786223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='ReadEegSpectr' href='#0'> Read EEG Spectrograms </a>\nIn version 4 onward, we use EEG spectrograms in addition to Kaggle spectrograms. The EEG spectrograms come from Kaggle dataset [here][4] (which were created from  spectrogram starter [here][5]). Thank you for upvoting dataset!\n\n[4]: https://www.kaggle.com/datasets/cdeotte/brain-eeg-spectrograms\n[5]: https://www.kaggle.com/code/cdeotte/how-to-make-spectrogram-from-eeg","metadata":{}},{"cell_type":"code","source":"%%time\n\nREAD_EEG_SPEC_FILES = False\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i%100==0: print(i,', ', end='')\n        x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy', allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:46:38.788376Z","iopub.execute_input":"2024-02-21T15:46:38.788662Z","iopub.status.idle":"2024-02-21T15:47:47.907669Z","shell.execute_reply.started":"2024-02-21T15:46:38.788636Z","shell.execute_reply":"2024-02-21T15:47:47.906738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='Loader' href='#0'> Data Loader  </a>\nThis dataloader outputs 4 spectrogram images as a 4 channel image of size 128x256x4 per train sample. This notebook version is not using data augmention but the code is available below to experiment with albumentations data augmention. Just add `augment = True` when creating the train data loader. And consider adding new transformations to the augment function below.\n\nUPDATE: In version 4 onward, our dataloader outputs both Kaggle spectrograms and EEG spectrogams as 8 channel image of size 128x256x8.","metadata":{"papermill":{"duration":0.010384,"end_time":"2024-01-14T22:52:47.341581","exception":false,"start_time":"2024-01-14T22:52:47.331197","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs = spectrograms, eeg_specs = all_eegs): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ))\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X)\n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data))\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test': \n                r = 0\n            else: \n                r = int( (row['min'] + row['max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n                \n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n            \n        return X,y\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch","metadata":{"papermill":{"duration":2.369789,"end_time":"2024-01-14T22:52:49.721728","exception":false,"start_time":"2024-01-14T22:52:47.351939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:47:47.909098Z","iopub.execute_input":"2024-02-21T15:47:47.909575Z","iopub.status.idle":"2024-02-21T15:47:47.929154Z","shell.execute_reply.started":"2024-02-21T15:47:47.90954Z","shell.execute_reply":"2024-02-21T15:47:47.928143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='Visual' href='#0'> Display Data </a>\nBelow we display example dataloader spectrogram images.","metadata":{"papermill":{"duration":0.00888,"end_time":"2024-01-14T22:52:49.739973","exception":false,"start_time":"2024-01-14T22:52:49.731093","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if CVSCORES:\n    gen = DataGenerator(train, batch_size=32, shuffle=False)\n    ROWS=2; COLS=3; BATCHES=2\n\n    for i,(x,y) in enumerate(gen):\n        plt.figure(figsize=(20,8))\n        for j in range(ROWS):\n            for k in range(COLS):\n                plt.subplot(ROWS,COLS,j*COLS+k+1)\n                t = y[j*COLS+k]\n                img = x[j*COLS+k,:,:,0][::-1,]\n                mn = img.flatten().min()\n                mx = img.flatten().max()\n                img = (img-mn)/(mx-mn)\n                plt.imshow(img)\n                tars = f'[{t[0]:0.2f}'\n                for s in t[1:]: tars += f', {s:0.2f}'\n                eeg = train.eeg_id.values[i*32+j*COLS+k]\n                plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n                plt.yticks([])\n                plt.ylabel('Frequencies (Hz)',size=14)\n                plt.xlabel('Time (sec)',size=16)\n        plt.show()\n        if i==BATCHES-1: break","metadata":{"papermill":{"duration":2.448242,"end_time":"2024-01-14T22:52:52.197249","exception":false,"start_time":"2024-01-14T22:52:49.749007","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:47:47.930492Z","iopub.execute_input":"2024-02-21T15:47:47.930867Z","iopub.status.idle":"2024-02-21T15:47:50.414695Z","shell.execute_reply.started":"2024-02-21T15:47:47.930836Z","shell.execute_reply":"2024-02-21T15:47:50.413809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='MODEL' href='#0'> Model </a>","metadata":{"papermill":{"duration":0.027228,"end_time":"2024-01-14T22:52:52.97653","exception":false,"start_time":"2024-01-14T22:52:52.949302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from keras.applications import  Xception \nfrom keras.applications import  ResNet50V2, ResNet101V2, ResNet152V2\nfrom keras.applications.efficientnet_v2 import EfficientNetV2B0, EfficientNetV2B1, EfficientNetV2B2, EfficientNetV2B3, EfficientNetV2S\nfrom keras.applications import ConvNeXtBase, ConvNeXtSmall, ConvNeXtTiny, ConvNeXtLarge, ConvNeXtXLarge","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:47:50.829178Z","iopub.execute_input":"2024-02-21T15:47:50.829486Z","iopub.status.idle":"2024-02-21T15:47:50.83465Z","shell.execute_reply.started":"2024-02-21T15:47:50.829436Z","shell.execute_reply":"2024-02-21T15:47:50.833677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def NetBase(Use, base):\n    use_model = None\n\n    if Use == \"Xception\":\n        \n        use_model = Xception(include_top=False, weights=None, input_shape=None)\n        use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/xception_weights_tf_dim_ordering_tf_kernels_notop.h5')\n        print(\"Build xception model \")\n            \n    if Use == \"Efficient\":\n        if base == 0:\n            use_model = EfficientNetV2B0(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/image-net-weights-group1/efficientnetv2-b0_notop.h5')\n\n        elif base == 1:\n            use_model = EfficientNetV2B1(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/image-net-weights-group1/efficientnetv2-b1_notop.h5')\n\n        elif base == 2:\n            use_model = EfficientNetV2B2(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/image-net-weights-group1/efficientnetv2-b2_notop.h5')\n        \n        elif base == 3:\n            use_model = EfficientNetV2B3(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/image-net-weights-group1/efficientnetv2-b3_notop.h5')\n            \n        elif base == 4:\n            use_model = EfficientNetV2S(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/image-net-weights-group1/efficientnetv2-s_notop.h5')\n\n    \n    if Use == \"ResNet\":\n        if base == 0:\n            use_model = ResNet50V2(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/resnet50v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n           \n        elif base == 1:\n            use_model = ResNet101V2(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/resnet101v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n        elif base == 2:\n            use_model = ResNet152V2(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/resnet152v2_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n    if Use == \"ConvNeXt\":\n        if base == 0:\n            use_model = ConvNeXtTiny(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/convnext_tiny_notop.h5')\n           \n        elif base == 1:\n            use_model = ConvNeXtSmall(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/convnext_small_notop.h5')\n\n        elif base == 2:\n            use_model = ConvNeXtBase(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/convnext_base_notop.h5')\n            \n        elif base == 3:\n            use_model = ConvNeXtLarge(include_top=False, weights=None, input_shape=None)\n            use_model.load_weights('/kaggle/input/resnetv2-imagenet-weights/convnext_large_notop.h5')\n            \n    return use_model","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:47:50.836139Z","iopub.execute_input":"2024-02-21T15:47:50.836426Z","iopub.status.idle":"2024-02-21T15:47:50.852323Z","shell.execute_reply.started":"2024-02-21T15:47:50.836402Z","shell.execute_reply":"2024-02-21T15:47:50.851394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    \n    inp = keras.Input(shape=(128,256,8))  \n    base_model = NetBase(TrainConfig['MODEL'], TrainConfig['BASE'])\n\n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    x1 = keras.layers.Concatenate(axis=1)(x1)\n    # EEG SPECTROGRAMS\n    x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    x2 = keras.layers.Concatenate(axis=1)(x2)\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = keras.layers.Concatenate(axis=2)([x1,x2])\n    elif USE_EEG_SPECTROGRAMS: x = x2\n    else: x = x1\n    x = keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x = base_model(x)\n    x = keras.layers.GlobalAveragePooling2D()(x)\n    x = keras.layers.Dense(6,activation='softmax')(x)\n    #x = keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = keras.Model(inputs=inp, outputs=x)\n    optim   = keras.optimizers.Adam(learning_rate = 1e-3)\n    loss  = keras.losses.KLDivergence()\n    metrics  =[keras.metrics.BinaryAccuracy(), keras.metrics.CategoricalAccuracy()]\n\n    model.compile(loss=loss, optimizer = optim, metrics=metrics) \n        \n    return model","metadata":{"papermill":{"duration":0.057714,"end_time":"2024-01-14T22:53:06.682056","exception":false,"start_time":"2024-01-14T22:53:06.624342","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:47:50.85341Z","iopub.execute_input":"2024-02-21T15:47:50.853711Z","iopub.status.idle":"2024-02-21T15:47:50.867904Z","shell.execute_reply.started":"2024-02-21T15:47:50.853686Z","shell.execute_reply":"2024-02-21T15:47:50.867209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='ScoreFunct' href='#0'> Score Function </a>","metadata":{}},{"cell_type":"code","source":"def fold_score(fold_oof, fold_true):\n    \"\"\"Kl Divergence Score\"\"\"\n    print(f\"Lengths: {len(fold_oof)} {len(fold_oof)}\")\n    oof = pd.DataFrame(fold_oof.copy())\n    oof['id'] = np.arange(len(oof))\n\n    true = pd.DataFrame(fold_true.copy())\n    true['id'] = np.arange(len(true))\n\n    cv = score(solution=true, submission=oof, row_id_column_name='id')\n    print('\\n CV Fold Score  =', cv)\n    return cv\n\ndef target_fold_score(fold_oof, fold_true):\n    \"\"\"Kl Divergence Score\"\"\"\n    #print(f\"\\n Lengths: {len(fold_oof)} {len(fold_oof)}\")\n    oof = pd.DataFrame(fold_oof.copy())\n    oof['id'] = np.arange(len(oof))\n\n    true = pd.DataFrame(fold_true.copy())\n    true['id'] = np.arange(len(true))\n    \n    fold_shapes = []\n    traget_score = []\n    for t in range(6):\n        t_true  = true[true[t] > 0].copy()\n        t_index = t_true.index\n        t_oof   = oof.iloc[t_index,:].copy()\n        t_true.reset_index()\n        t_oof.reset_index()\n        t_oof['id'] = np.arange(len(t_oof))\n        t_true['id'] = np.arange(len(t_true))\n        assert(t_oof.id.values.all() == t_true.id.values.all())\n        fold_shape = t_true.shape\n        fold_shapes.append(fold_shape)\n\n        t_cv = score(solution=t_true, submission=t_oof, row_id_column_name='id')\n        traget_score.append(t_cv)\n        print(f\" Target: {t} \\t {t_true.shape} \\t {t_cv}\")\n    \n    cv = score(solution=true, submission=oof, row_id_column_name='id')\n    print('\\n CV Fold Score  =', cv)\n    return cv","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:47:50.868957Z","iopub.execute_input":"2024-02-21T15:47:50.869244Z","iopub.status.idle":"2024-02-21T15:47:50.883184Z","shell.execute_reply.started":"2024-02-21T15:47:50.86922Z","shell.execute_reply":"2024-02-21T15:47:50.882282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='Eval' href='#0'> Evaluate Model </a>\n","metadata":{"papermill":{"duration":0.033717,"end_time":"2024-01-14T22:53:06.742557","exception":false,"start_time":"2024-01-14T22:53:06.70884","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, TerminateOnNaN, EarlyStopping, LambdaCallback\nfrom keras.models import save_model, load_model\nimport gc\nimport json","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:47:50.884333Z","iopub.execute_input":"2024-02-21T15:47:50.884661Z","iopub.status.idle":"2024-02-21T15:47:50.89692Z","shell.execute_reply.started":"2024-02-21T15:47:50.884631Z","shell.execute_reply":"2024-02-21T15:47:50.896156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nSCORE = []\nFOLD_SCORE = []\nall_oof = []\nall_true = []\nprint(f\"\\n Using {TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}. Trained using Batch Size {TrainConfig['TRAIN_BATCH']} and {TrainConfig['NSPLIT']} folds\")\n\nif CVSCORES:\n    gkf = GroupKFold(n_splits=TrainConfig['NSPLIT'])\n    for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n\n        print('#'*25)\n        print(f\"### Fold {i+1}/{TrainConfig['NSPLIT']}\")\n        #train_gen = DataGenerator(train.iloc[train_index], shuffle=True,  batch_size=32, augment=False)\n        valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n\n        keras.backend.clear_session()\n\n        print(f'### Loading Saved Model')\n        print('#'*25)\n        with strategy.scope():\n            model = build_model()\n        try:\n            ############  Adjust to model saving format  ################\n            model.load_weights(f\"{LOAD_MODELS_FROM}{TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}_Fold_Check_{i}.keras\")\n        except:\n            print(f'### Model weights not Found')\n            break\n\n        with strategy.scope():\n            oof   = model.predict(valid_gen, verbose=1)\n        try:\n            fold_true = train.iloc[valid_index][TARGETS].values\n            flodScore = target_fold_score(oof, fold_true)\n            FOLD_SCORE.append(flodScore)\n        except:\n            print(\"Problem Calculating Score Here \")\n            break\n\n\n        all_oof.append(oof)\n        all_true.append(train.iloc[valid_index][TARGETS].values)\n\n        try:\n            SCORE.append(fold_score(np.concatenate(all_oof), np.concatenate(all_true)))\n        except:\n            print(\"Problem Calculating Score Here \")\n\n        del model, oof\n        gc.collect()\n\n    print(\"Finished Training\")\n    print(\"Fold Score :\", flodScore)\n\n    all_oof      = np.concatenate(all_oof)\n    all_true     = np.concatenate(all_true)","metadata":{"papermill":{"duration":161.172,"end_time":"2024-01-14T22:55:47.94776","exception":false,"start_time":"2024-01-14T22:53:06.77576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:47:50.898236Z","iopub.execute_input":"2024-02-21T15:47:50.898518Z","iopub.status.idle":"2024-02-21T15:53:10.118961Z","shell.execute_reply.started":"2024-02-21T15:47:50.898493Z","shell.execute_reply":"2024-02-21T15:53:10.117919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='Save' href='#0'> Update & save Config </a>","metadata":{}},{"cell_type":"code","source":"if CVSCORES:\n    TrainConfig['FOLD_SCORE'] = FOLD_SCORE\n    TrainConfig['SCORE']      = SCORE\n    TrainConfig['RUN_DATE']   = RUN_DATE\n    TrainConfig['INFER_ID']   = RUNID\n    \n    yaml.dump(TrainConfig, open(f\"{SAVE_PATH}/{TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}_RUN_{RUNID}_Config.yaml\", 'w'), Dumper=Dumper)\n    \n    SavedRunConfig = yaml.load( open(f\"{SAVE_PATH}/{TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}_RUN_{RUNID}_Config.yaml\", 'r'), Loader=Loader)\n    print(SavedRunConfig['SCORE'])","metadata":{"execution":{"iopub.status.busy":"2024-02-21T16:09:01.84351Z","iopub.execute_input":"2024-02-21T16:09:01.844268Z","iopub.status.idle":"2024-02-21T16:09:01.853735Z","shell.execute_reply.started":"2024-02-21T16:09:01.844236Z","shell.execute_reply":"2024-02-21T16:09:01.852914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CVSCORES:\n    print(SCORE)\n    print(FOLD_SCORE , \"\\t\",  np.mean(FOLD_SCORE))","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:53:10.166298Z","iopub.execute_input":"2024-02-21T15:53:10.166625Z","iopub.status.idle":"2024-02-21T15:53:10.175154Z","shell.execute_reply.started":"2024-02-21T15:53:10.166594Z","shell.execute_reply":"2024-02-21T15:53:10.17434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='Score' href='#0'> CV Score </a>","metadata":{"papermill":{"duration":0.047893,"end_time":"2024-01-14T22:55:48.045405","exception":false,"start_time":"2024-01-14T22:55:47.997512","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if CVSCORES:\n    try:\n        oof = pd.DataFrame(all_oof.copy())\n        oof['id'] = np.arange(len(oof))\n        true = pd.DataFrame(all_true.copy())\n        true['id'] = np.arange(len(true))\n\n        print(\"\\n \", len(oof),\"Colmns: \", oof.columns, len(true), \"Colmns: \", true.columns)\n\n        cv = score(solution=true, submission=oof, row_id_column_name='id')\n        print(f\"CV Score KL-Div for {TrainConfig['MODEL']}{TrainConfig['BASE']} = {cv}\")\n    except:\n        print(\"Cant do it\")","metadata":{"papermill":{"duration":0.126007,"end_time":"2024-01-14T22:55:48.222599","exception":false,"start_time":"2024-01-14T22:55:48.096592","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:53:10.176332Z","iopub.execute_input":"2024-02-21T15:53:10.17673Z","iopub.status.idle":"2024-02-21T15:53:10.237993Z","shell.execute_reply.started":"2024-02-21T15:53:10.176699Z","shell.execute_reply":"2024-02-21T15:53:10.237174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## <a id ='Infer' href='#0'> Infer on Test data </a>","metadata":{"papermill":{"duration":0.050491,"end_time":"2024-01-14T22:55:48.321932","exception":false,"start_time":"2024-01-14T22:55:48.271441","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#del all_eegs, spectrograms;\ngc.collect()\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"papermill":{"duration":0.073698,"end_time":"2024-01-14T22:55:48.445914","exception":false,"start_time":"2024-01-14T22:55:48.372216","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:53:10.23929Z","iopub.execute_input":"2024-02-21T15:53:10.239667Z","iopub.status.idle":"2024-02-21T15:53:10.468621Z","shell.execute_reply.started":"2024-02-21T15:53:10.239637Z","shell.execute_reply":"2024-02-21T15:53:10.467504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"papermill":{"duration":0.257975,"end_time":"2024-01-14T22:55:48.757931","exception":false,"start_time":"2024-01-14T22:55:48.499956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:53:10.469795Z","iopub.execute_input":"2024-02-21T15:53:10.470079Z","iopub.status.idle":"2024-02-21T15:53:10.819112Z","shell.execute_reply.started":"2024-02-21T15:53:10.470054Z","shell.execute_reply":"2024-02-21T15:53:10.818119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-21T15:53:10.821224Z","iopub.execute_input":"2024-02-21T15:53:10.821558Z","iopub.status.idle":"2024-02-21T15:53:10.932188Z","shell.execute_reply.started":"2024-02-21T15:53:10.821531Z","shell.execute_reply":"2024-02-21T15:53:10.931109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-02-21T15:53:10.933847Z","iopub.execute_input":"2024-02-21T15:53:10.935342Z","iopub.status.idle":"2024-02-21T15:53:22.945421Z","shell.execute_reply.started":"2024-02-21T15:53:10.935304Z","shell.execute_reply":"2024-02-21T15:53:22.944287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER EFFICIENTNET ON TEST\npreds = []\nwpreds = []\nmodel = build_model()\ntest_gen = DataGenerator(test, shuffle=False, batch_size=64, mode='test',\n                         specs = spectrograms2, eeg_specs = all_eegs2)\nif CVSCORES:\n    \n    score_ratio = FOLD_SCORE/np.sum(FOLD_SCORE, axis=0)\n\n\nfor i in range(TrainConfig['NSPLIT']):\n    print(f'Fold {i+1}')\n    if LOAD_MODELS_FROM:\n            try:\n                model.load_weights(f\"{LOAD_MODELS_FROM}{TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}_Fold_Check_{i}.keras\")\n            \n            except:\n                print(\"Cant Find Model weights\")\n                break\n    else:\n        model.load_weights(f\"{SAVE_PATH}/{TrainConfig['MODEL']}_{TrainConfig['BASE']}_{TrainConfig['VER']}_Fold_Check_{i}.keras\")\n    \n    with strategy.scope():\n        pred = model.predict(test_gen, verbose = 1)\n        preds.append(pred)\n\npred   = np.mean(preds,axis=0)\n\nprint()\nprint('Test preds shape', pred.shape)\n","metadata":{"papermill":{"duration":9.827745,"end_time":"2024-01-14T22:55:58.637732","exception":false,"start_time":"2024-01-14T22:55:48.809987","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:53:22.946605Z","iopub.execute_input":"2024-02-21T15:53:22.947158Z","iopub.status.idle":"2024-02-21T15:54:04.702627Z","shell.execute_reply.started":"2024-02-21T15:53:22.947131Z","shell.execute_reply":"2024-02-21T15:54:04.701607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id ='Submit' href='#0'> Submit </a>","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = pred\n\nsub.to_csv('submission.csv',index=False)\nprint('Submissionn shape',sub.shape)\ndisplay(sub.head())","metadata":{"papermill":{"duration":0.071388,"end_time":"2024-01-14T22:55:58.760368","exception":false,"start_time":"2024-01-14T22:55:58.68898","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:54:04.736598Z","iopub.execute_input":"2024-02-21T15:54:04.736868Z","iopub.status.idle":"2024-02-21T15:54:04.779145Z","shell.execute_reply.started":"2024-02-21T15:54:04.736845Z","shell.execute_reply":"2024-02-21T15:54:04.778217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsub.iloc[:,-6:].sum(axis=1)","metadata":{"papermill":{"duration":0.062742,"end_time":"2024-01-14T22:55:58.873394","exception":false,"start_time":"2024-01-14T22:55:58.810652","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-02-21T15:54:04.780444Z","iopub.execute_input":"2024-02-21T15:54:04.780742Z","iopub.status.idle":"2024-02-21T15:54:04.789188Z","shell.execute_reply.started":"2024-02-21T15:54:04.780716Z","shell.execute_reply":"2024-02-21T15:54:04.788135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- TPU Train  [notebook](https://www.kaggle.com/mcwema/hms-hbac-train-tf2-15-keras3-0-2-tpu/edit)\n- credit this [notebook](https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43)","metadata":{}}]}