{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7403069,"sourceType":"datasetVersion","datasetId":4304949},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":7465251,"sourceType":"datasetVersion","datasetId":4317718},{"sourceId":7570342,"sourceType":"datasetVersion","datasetId":4407194},{"sourceId":7581697,"sourceType":"datasetVersion","datasetId":4413439},{"sourceId":7581715,"sourceType":"datasetVersion","datasetId":4413451},{"sourceId":7581720,"sourceType":"datasetVersion","datasetId":4413454},{"sourceId":7626715,"sourceType":"datasetVersion","datasetId":4382744},{"sourceId":7658551,"sourceType":"datasetVersion","datasetId":4417235},{"sourceId":158958765,"sourceType":"kernelVersion"},{"sourceId":159333316,"sourceType":"kernelVersion"},{"sourceId":159396114,"sourceType":"kernelVersion"}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Model 1","metadata":{}},{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n\nVER = 5\n\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = '/kaggle/input/brain-efficientnet-models-v3-v4-v5/'\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:15:09.269904Z","iopub.execute_input":"2024-02-21T01:15:09.270299Z","iopub.status.idle":"2024-02-21T01:15:23.422666Z","shell.execute_reply.started":"2024-02-21T01:15:09.270268Z","shell.execute_reply":"2024-02-21T01:15:23.421696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:15:23.424611Z","iopub.execute_input":"2024-02-21T01:15:23.425124Z","iopub.status.idle":"2024-02-21T01:15:23.430010Z","shell.execute_reply.started":"2024-02-21T01:15:23.425098Z","shell.execute_reply":"2024-02-21T01:15:23.429147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:15:23.431275Z","iopub.execute_input":"2024-02-21T01:15:23.431772Z","iopub.status.idle":"2024-02-21T01:15:23.724004Z","shell.execute_reply.started":"2024-02-21T01:15:23.431748Z","shell.execute_reply":"2024-02-21T01:15:23.723028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:15:23.725356Z","iopub.execute_input":"2024-02-21T01:15:23.725718Z","iopub.status.idle":"2024-02-21T01:15:23.820604Z","shell.execute_reply.started":"2024-02-21T01:15:23.725686Z","shell.execute_reply":"2024-02-21T01:15:23.819707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nREAD_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:15:23.823649Z","iopub.execute_input":"2024-02-21T01:15:23.823922Z","iopub.status.idle":"2024-02-21T01:16:23.405788Z","shell.execute_reply.started":"2024-02-21T01:15:23.823899Z","shell.execute_reply":"2024-02-21T01:16:23.404678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nREAD_EEG_SPEC_FILES = False\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i%100==0: print(i,', ',end='')\n        x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:16:23.407063Z","iopub.execute_input":"2024-02-21T01:16:23.407411Z","iopub.status.idle":"2024-02-21T01:17:34.675069Z","shell.execute_reply.started":"2024-02-21T01:16:23.407380Z","shell.execute_reply":"2024-02-21T01:17:34.674081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as albu\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs = spectrograms, eeg_specs = all_eegs): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X) \n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test': \n                r = 0\n            else: \n                r = int( (row['min'] + row['max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n                \n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n            \n        return X,y\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:34.676112Z","iopub.execute_input":"2024-02-21T01:17:34.676391Z","iopub.status.idle":"2024-02-21T01:17:36.801157Z","shell.execute_reply.started":"2024-02-21T01:17:34.676367Z","shell.execute_reply":"2024-02-21T01:17:36.799906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen = DataGenerator(train, batch_size=32, shuffle=False)\nROWS=2; COLS=3; BATCHES=2\n\nfor i,(x,y) in enumerate(gen):\n    plt.figure(figsize=(20,8))\n    for j in range(ROWS):\n        for k in range(COLS):\n            plt.subplot(ROWS,COLS,j*COLS+k+1)\n            t = y[j*COLS+k]\n            img = x[j*COLS+k,:,:,0][::-1,]\n            mn = img.flatten().min()\n            mx = img.flatten().max()\n            img = (img-mn)/(mx-mn)\n            plt.imshow(img)\n            tars = f'[{t[0]:0.2f}'\n            for s in t[1:]: tars += f', {s:0.2f}'\n            eeg = train.eeg_id.values[i*32+j*COLS+k]\n            plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n            plt.yticks([])\n            plt.ylabel('Frequencies (Hz)',size=14)\n            plt.xlabel('Time (sec)',size=16)\n    plt.show()\n    if i==BATCHES-1: break","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:36.802519Z","iopub.execute_input":"2024-02-21T01:17:36.803076Z","iopub.status.idle":"2024-02-21T01:17:39.532936Z","shell.execute_reply.started":"2024-02-21T01:17:36.803044Z","shell.execute_reply":"2024-02-21T01:17:39.531909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nLR_START = 1e-6\nLR_MAX = 1e-3\nLR_MIN = 1e-6\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS2 = 10\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS2)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y, '-o')\nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Cosine Training Schedule',size=16); plt.show()\n\nLR2 = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:39.534192Z","iopub.execute_input":"2024-02-21T01:17:39.534481Z","iopub.status.idle":"2024-02-21T01:17:39.811307Z","shell.execute_reply.started":"2024-02-21T01:17:39.534456Z","shell.execute_reply":"2024-02-21T01:17:39.810394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 1e-4\nLR_MAX = 1e-3\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.1\nEVERY = 1\nEPOCHS = 4\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//EVERY)\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, y, 'o-'); \nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Step Training Schedule',size=16); plt.show()\n\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:39.812390Z","iopub.execute_input":"2024-02-21T01:17:39.812665Z","iopub.status.idle":"2024-02-21T01:17:40.034237Z","shell.execute_reply.started":"2024-02-21T01:17:39.812641Z","shell.execute_reply":"2024-02-21T01:17:40.033293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:40.035420Z","iopub.execute_input":"2024-02-21T01:17:40.035708Z","iopub.status.idle":"2024-02-21T01:17:53.785000Z","shell.execute_reply.started":"2024-02-21T01:17:40.035684Z","shell.execute_reply":"2024-02-21T01:17:53.783937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\ndef build_model():\n    \n    inp = tf.keras.Input(shape=(128,256,8))\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n    \n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n    # EEG SPECTROGRAMS\n    x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = tf.keras.layers.Concatenate(axis=2)([x1,x2])\n    elif USE_EEG_SPECTROGRAMS: x = x2\n    else: x = x1\n    x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt) \n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:53.786442Z","iopub.execute_input":"2024-02-21T01:17:53.786756Z","iopub.status.idle":"2024-02-21T01:17:53.809646Z","shell.execute_reply.started":"2024-02-21T01:17:53.786727Z","shell.execute_reply":"2024-02-21T01:17:53.808774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\nall_oof = []\nall_true = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=False)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'EffNet_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n        \n    oof = model.predict(valid_gen, verbose=1)\n    all_oof.append(oof)\n    all_true.append(train.iloc[valid_index][TARGETS].values)\n    \n    del model, oof\n    gc.collect()\n    \nall_oof = np.concatenate(all_oof)\nall_true = np.concatenate(all_true)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:17:53.810724Z","iopub.execute_input":"2024-02-21T01:17:53.810988Z","iopub.status.idle":"2024-02-21T01:20:53.733674Z","shell.execute_reply.started":"2024-02-21T01:17:53.810966Z","shell.execute_reply":"2024-02-21T01:20:53.732866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\noof = pd.DataFrame(all_oof.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score KL-Div for EfficientNetB2 =',cv)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:20:53.739165Z","iopub.execute_input":"2024-02-21T01:20:53.739487Z","iopub.status.idle":"2024-02-21T01:20:53.815039Z","shell.execute_reply.started":"2024-02-21T01:20:53.739461Z","shell.execute_reply":"2024-02-21T01:20:53.814181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del all_eegs, spectrograms; gc.collect()\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:20:53.816072Z","iopub.execute_input":"2024-02-21T01:20:53.816349Z","iopub.status.idle":"2024-02-21T01:20:54.031217Z","shell.execute_reply.started":"2024-02-21T01:20:53.816326Z","shell.execute_reply":"2024-02-21T01:20:54.030199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:20:54.032285Z","iopub.execute_input":"2024-02-21T01:20:54.032563Z","iopub.status.idle":"2024-02-21T01:20:54.306933Z","shell.execute_reply.started":"2024-02-21T01:20:54.032539Z","shell.execute_reply":"2024-02-21T01:20:54.306185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:20:54.308071Z","iopub.execute_input":"2024-02-21T01:20:54.308354Z","iopub.status.idle":"2024-02-21T01:20:54.332144Z","shell.execute_reply.started":"2024-02-21T01:20:54.308330Z","shell.execute_reply":"2024-02-21T01:20:54.331354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:20:54.333462Z","iopub.execute_input":"2024-02-21T01:20:54.333737Z","iopub.status.idle":"2024-02-21T01:21:06.140620Z","shell.execute_reply.started":"2024-02-21T01:20:54.333714Z","shell.execute_reply":"2024-02-21T01:21:06.139541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER EFFICIENTNET ON TEST\npreds = []\nmodel = build_model()\ntest_gen = DataGenerator(test, shuffle=False, batch_size=64, mode='test',\n                         specs = spectrograms2, eeg_specs = all_eegs2)\n\nfor i in range(5):\n    print(f'Fold {i+1}')\n    if LOAD_MODELS_FROM:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'EffNet_v{VER}_f{i}.h5')\n    pred = model.predict(test_gen, verbose=1)\n    preds.append(pred)\npred = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:06.141888Z","iopub.execute_input":"2024-02-21T01:21:06.142463Z","iopub.status.idle":"2024-02-21T01:21:13.384371Z","shell.execute_reply.started":"2024-02-21T01:21:06.142436Z","shell.execute_reply":"2024-02-21T01:21:13.383389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1 = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub1[TARGETS] = pred\nprint('Submission shape',sub1.shape)\nsub1.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:13.385515Z","iopub.execute_input":"2024-02-21T01:21:13.385809Z","iopub.status.idle":"2024-02-21T01:21:13.401192Z","shell.execute_reply.started":"2024-02-21T01:21:13.385784Z","shell.execute_reply":"2024-02-21T01:21:13.400251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_combine = pred\npreds_combine","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:13.402668Z","iopub.execute_input":"2024-02-21T01:21:13.403489Z","iopub.status.idle":"2024-02-21T01:21:13.411932Z","shell.execute_reply.started":"2024-02-21T01:21:13.403456Z","shell.execute_reply":"2024-02-21T01:21:13.411067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsub1.iloc[:,-6:].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:13.413243Z","iopub.execute_input":"2024-02-21T01:21:13.413642Z","iopub.status.idle":"2024-02-21T01:21:13.426718Z","shell.execute_reply.started":"2024-02-21T01:21:13.413608Z","shell.execute_reply":"2024-02-21T01:21:13.425877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2","metadata":{}},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# import torch \n# import torch.nn as nn\n# import torch.nn.functional as F\n# import torchvision.transforms as transforms\n# import random\n# import warnings\n# warnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:13.427907Z","iopub.execute_input":"2024-02-21T01:21:13.428288Z","iopub.status.idle":"2024-02-21T01:21:17.103930Z","shell.execute_reply.started":"2024-02-21T01:21:13.428258Z","shell.execute_reply":"2024-02-21T01:21:17.102857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Config:\n#     seed=2024\n#     image_transform=transforms.Resize((512, 512))\n#     num_folds=5","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:17.105545Z","iopub.execute_input":"2024-02-21T01:21:17.106480Z","iopub.status.idle":"2024-02-21T01:21:17.111933Z","shell.execute_reply.started":"2024-02-21T01:21:17.106441Z","shell.execute_reply":"2024-02-21T01:21:17.110470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models=[]\n# for i in range(Config.num_folds):\n#     model = torch.load(f'/kaggle/input/hms-baseline-resnet34d-512-512-training-5-folds/HMS_resnet_fold{i}.pth')\n#     models.append(model)\n# model = torch.load(\"/kaggle/input/hms-baseline-resnet34d-512-512-training/HMS_resnet.pth\")\n# models.append(model)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:17.113416Z","iopub.execute_input":"2024-02-21T01:21:17.113756Z","iopub.status.idle":"2024-02-21T01:21:23.486938Z","shell.execute_reply.started":"2024-02-21T01:21:17.113724Z","shell.execute_reply":"2024-02-21T01:21:23.486077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def seed_everything(seed):\n#     torch.backends.cudnn.deterministic = True#将cuda加速的随机数生成器设为确定性模式\n#     torch.backends.cudnn.benchmark = True#关闭CuDNN框架的自动寻找最优卷积算法的功能，以避免不同的算法对结果产生影响\n#     torch.manual_seed(seed)#pytorch的随机种子\n#     np.random.seed(seed)#numpy的随机种子\n#     random.seed(seed)#python内置的随机种子\n# seed_everything(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:23.488171Z","iopub.execute_input":"2024-02-21T01:21:23.488473Z","iopub.status.idle":"2024-02-21T01:21:23.498955Z","shell.execute_reply.started":"2024-02-21T01:21:23.488447Z","shell.execute_reply":"2024-02-21T01:21:23.498068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\n# submission=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\n# submission=submission.merge(test_df,on='eeg_id',how='left')\n# submission['path']=submission['spectrogram_id'].apply(lambda x: \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\"+str(x)+\".parquet\" )\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:23.500440Z","iopub.execute_input":"2024-02-21T01:21:23.500778Z","iopub.status.idle":"2024-02-21T01:21:23.536415Z","shell.execute_reply.started":"2024-02-21T01:21:23.500748Z","shell.execute_reply":"2024-02-21T01:21:23.535480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# paths=submission['path'].values\n# test_preds=[]\n# for path in paths:\n#     eps=1e-6\n#     data=pd.read_parquet(path)\n#     #这里最小值是0,故用-1填充.第一列是时间列,故去掉 ,行是不同列,列是时间\n#     data = data.fillna(-1).values[:,1:].T\n#     #选取一段时间的数据进行训练\n#     data=data[:,0:300]#(400,300)\n#     data=np.clip(data,np.exp(-6),np.exp(10))#最大值为89209464.0\n#     data= np.log(data)#对数变换\n#     #对数据进行归一化\n#     data_mean=data.mean(axis=(0,1))\n#     data_std=data.std(axis=(0,1))\n#     data=(data-data_mean)/(data_std+eps)\n#     data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n#     data=Config.image_transform(data_tensor)\n#     test_pred=[]\n#     for model in models:\n#         model.eval()\n#         with torch.no_grad():\n#             pred=F.softmax(model(data.unsqueeze(0)))[0]\n#             pred=pred.detach().cpu().numpy()\n#         test_pred.append(pred)\n#     test_pred=np.array(test_pred).mean(axis=0)\n#     test_preds.append(test_pred)\n# test_preds=np.array(test_preds)\n# test_preds","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:23.537798Z","iopub.execute_input":"2024-02-21T01:21:23.538524Z","iopub.status.idle":"2024-02-21T01:21:25.684877Z","shell.execute_reply.started":"2024-02-21T01:21:23.538485Z","shell.execute_reply":"2024-02-21T01:21:25.683724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub2=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\n# labels=['seizure','lpd','gpd','lrda','grda','other']\n# for i in range(len(labels)):\n#     sub2[f'{labels[i]}_vote']=test_preds[:,i]\n# sub2.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.686440Z","iopub.execute_input":"2024-02-21T01:21:25.686899Z","iopub.status.idle":"2024-02-21T01:21:25.707512Z","shell.execute_reply.started":"2024-02-21T01:21:25.686857Z","shell.execute_reply":"2024-02-21T01:21:25.706534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3","metadata":{}},{"cell_type":"code","source":"# # Importing essential libraries\n# import gc\n# import os\n# import random\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from IPython.display import display\n\n# # PyTorch for deep learning\n# import timm\n# import torch\n# import torch.nn as nn  \n# import torch.optim as optim\n# import torch.nn.functional as F\n\n# # torchvision for image processing and augmentation\n# import torchvision.transforms as transforms\n\n# # Suppressing minor warnings to keep the output clean\n# warnings.filterwarnings('ignore', category=Warning)\n\n# # Reclaim memory no longer in use.\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.709554Z","iopub.execute_input":"2024-02-21T01:21:25.709864Z","iopub.status.idle":"2024-02-21T01:21:25.714402Z","shell.execute_reply.started":"2024-02-21T01:21:25.709838Z","shell.execute_reply":"2024-02-21T01:21:25.713437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Config:\n#     seed=42\n#     image_transform=transforms.Resize((512, 512))\n#     num_folds=5\n    \n# # Set the seed for reproducibility across multiple libraries\n# def set_seed(seed):\n#     torch.backends.cudnn.deterministic = True\n#     torch.backends.cudnn.benchmark = True\n#     torch.manual_seed(seed)\n#     np.random.seed(seed)\n#     random.seed(seed)\n    \n# set_seed(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.716188Z","iopub.execute_input":"2024-02-21T01:21:25.716530Z","iopub.status.idle":"2024-02-21T01:21:25.726696Z","shell.execute_reply.started":"2024-02-21T01:21:25.716504Z","shell.execute_reply":"2024-02-21T01:21:25.725844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load and store the trained models for each fold into a list\n# models = []\n\n# # Load ResNet34d\n# for i in range(Config.num_folds):\n#     # Create the same model architecture as during training\n#     model_resnet = timm.create_model('resnet34d', pretrained=False, num_classes=6, in_chans=1)\n    \n#     # Load the trained weights from the corresponding file\n#     model_resnet.load_state_dict(torch.load(f'/kaggle/input/resnet34d/hms-train-resnet34d/resnet34d_fold{i}.pth', map_location=torch.device('cpu')))\n    \n#     # Append the loaded model to the models list\n#     models.append(model_resnet)\n\n# # Reclaim memory no longer in use.\n# gc.collect()\n\n# # Load EfficientNetB0\n# for j in range(Config.num_folds):\n#     # Create the same model architecture as during training\n#     model_effnet_b0 = timm.create_model('efficientnet_b0', pretrained=False, num_classes=6, in_chans=1)\n    \n#     # Load the trained weights from the corresponding file\n#     model_effnet_b0.load_state_dict(torch.load(f'/kaggle/input/efficientnetb0/hms-train-efficientnetb0/efficientnet_b0_fold{j}.pth', map_location=torch.device('cpu')))\n    \n#     # Append the loaded model to the models list\n#     models.append(model_effnet_b0)\n    \n# # Reclaim memory no longer in use.\n# gc.collect()\n    \n# # Load EfficientNetB1\n# for k in range(Config.num_folds):\n#     # Create the same model architecture as during training\n#     model_effnet_b1 = timm.create_model('efficientnet_b1', pretrained=False, num_classes=6, in_chans=1)\n    \n#     # Load the trained weights from the corresponding file\n#     model_effnet_b1.load_state_dict(torch.load(f'/kaggle/input/efficientnetb1/hms-train-efficientnetb1/efficientnet_b1_fold{k}.pth', map_location=torch.device('cpu')))\n    \n#     # Append the loaded model to the models list\n#     models.append(model_effnet_b1)\n\n# # Reclaim memory no longer in use.\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.727731Z","iopub.execute_input":"2024-02-21T01:21:25.727996Z","iopub.status.idle":"2024-02-21T01:21:25.736672Z","shell.execute_reply.started":"2024-02-21T01:21:25.727974Z","shell.execute_reply":"2024-02-21T01:21:25.735751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load test data and sample submission dataframe\n# test_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\n# submission = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\n\n# # Merge the submission dataframe with the test data on EEG IDs\n# submission = submission.merge(test_df, on='eeg_id', how='left')\n\n# # Generate file paths for each spectrogram based on the EEG data in the submission dataframe\n# submission['path'] = submission['spectrogram_id'].apply(lambda x: f\"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/{x}.parquet\")\n\n# # Display the first few rows of the submission dataframe\n# display(submission.head())\n\n# # Reclaim memory no longer in use\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.737913Z","iopub.execute_input":"2024-02-21T01:21:25.738249Z","iopub.status.idle":"2024-02-21T01:21:25.749057Z","shell.execute_reply.started":"2024-02-21T01:21:25.738224Z","shell.execute_reply":"2024-02-21T01:21:25.748174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define the weights for each model\n# weight_resnet34d = 0.26\n# weight_effnetb0 = 0.48\n# weight_effnetb1 = 0.26\n\n# # Get file paths for test spectrograms\n# paths = submission['path'].values\n# test_predss = []\n\n# # Generate predictions for each spectrogram using all models\n# for path in paths:\n#     eps = 1e-6\n#     # Read and preprocess spectrogram data\n#     data = pd.read_parquet(path)\n#     data = data.fillna(-1).values[:, 1:].T\n#     data = np.clip(data, np.exp(-6), np.exp(10))\n#     data = np.log(data)\n    \n#     # Normalize the data\n#     data_mean = data.mean(axis=(0, 1))\n#     data_std = data.std(axis=(0, 1))\n#     data = (data - data_mean) / (data_std + eps)\n#     data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n#     data = Config.image_transform(data_tensor)\n\n#     test_pred = []\n    \n#     # Generate predictions using all models\n#     for model in models:\n#         model.eval()\n#         with torch.no_grad():\n#             pred = F.softmax(model(data.unsqueeze(0)))[0]\n#             pred = pred.detach().cpu().numpy()\n#         test_pred.append(pred)\n        \n#     # Combine predictions from all models using weighted voting\n#     weighted_pred = weight_resnet34d * np.mean(test_pred[:Config.num_folds], axis=0) + \\\n#                      weight_effnetb0 * np.mean(test_pred[Config.num_folds:2*Config.num_folds], axis=0) + \\\n#                      weight_effnetb1 * np.mean(test_pred[2*Config.num_folds:], axis=0)\n    \n#     test_predss.append(weighted_pred)\n\n# # Convert the list of predictions to a NumPy array for further processing\n# test_predss = np.array(test_predss)\n\n# # Reclaim memory no longer in use\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.750159Z","iopub.execute_input":"2024-02-21T01:21:25.750410Z","iopub.status.idle":"2024-02-21T01:21:25.763233Z","shell.execute_reply.started":"2024-02-21T01:21:25.750388Z","shell.execute_reply":"2024-02-21T01:21:25.762190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_predss","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.764392Z","iopub.execute_input":"2024-02-21T01:21:25.764663Z","iopub.status.idle":"2024-02-21T01:21:25.775614Z","shell.execute_reply.started":"2024-02-21T01:21:25.764640Z","shell.execute_reply":"2024-02-21T01:21:25.774749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 4","metadata":{}},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nimport tensorflow\nimport tensorflow.keras.backend as K\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.models import load_model\n\nLOAD_BACKBONE_FROM = '/kaggle/input/efficientnetb-tf-keras/EfficientNetB2.h5'\nLOAD_MODELS_FROM = '/kaggle/input/features-head-starter-models/'\nVER = 35\nDATA_TYPE = 'both' # both|eeg|kaggle|raw\nTEST_MODE = False\nsubmission = True\n\n# Setup for ensemble\nENSEMBLE = True\nLBs = [0.39,0.41,0.43,0.52] # for weighted ensemble we use LBs of each model\nVERK = 33.2 # Kaggle's spectrogram model version\nVERB = 35 # Kaggle's and EEG's spectrogram model version\nVERE = 34 # EEG's spectrogram model version\nVERR = 36 # EEG's raw wavenet model version\n\nnp.random.seed(42)\n\n# USE SINGLE GPU, MULTIPLE GPUS \ngpus = tf.config.list_physical_devices('GPU')\n# WE USE MIXED PRECISION\ntf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\nif len(gpus)>1:\n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\nelse:\n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.776880Z","iopub.execute_input":"2024-02-21T01:21:25.777245Z","iopub.status.idle":"2024-02-21T01:21:25.791632Z","shell.execute_reply.started":"2024-02-21T01:21:25.777220Z","shell.execute_reply":"2024-02-21T01:21:25.790690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nFEATS2 = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\nFEAT2IDX = {x:y for x,y in zip(FEATS2,range(len(FEATS2)))}\n\ndef eeg_from_parquet(parquet_path):\n\n    eeg = pd.read_parquet(parquet_path, columns=FEATS2)\n    rows = len(eeg)\n    offset = (rows-10_000)//2\n    eeg = eeg.iloc[offset:offset+10_000]\n    data = np.zeros((10_000,len(FEATS2)))\n    for j,col in enumerate(FEATS2):\n        \n        # FILL NAN\n        x = eeg[col].values.astype('float32')\n        m = np.nanmean(x)\n        if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n        else: x[:] = 0\n        \n        data[:,j] = x\n\n    return data\n\ndef add_kl(data):\n    import torch\n    labels = data[TARGETS].values + 1e-5\n\n    # compute kl-loss with uniform distribution by pytorch\n    data['kl'] = torch.nn.functional.kl_div(\n        torch.log(torch.tensor(labels)),\n        torch.tensor([1 / 6] * 6),\n        reduction='none'\n    ).sum(dim=1).numpy()\n    return data\n\nif not submission:\n    train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n    TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n    META = ['spectrogram_id','spectrogram_label_offset_seconds','patient_id','expert_consensus']\n    train = train.groupby('eeg_id')[META+TARGETS\n                           ].agg({**{m:'first' for m in META},**{t:'sum' for t in TARGETS}}).reset_index() \n    train[TARGETS] = train[TARGETS]/train[TARGETS].values.sum(axis=1,keepdims=True)\n    train.columns = ['eeg_id','spec_id','offset','patient_id','target'] + TARGETS\n    train = add_kl(train)\n    print(train.head(1).to_string())","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.792947Z","iopub.execute_input":"2024-02-21T01:21:25.793592Z","iopub.status.idle":"2024-02-21T01:21:25.807109Z","shell.execute_reply.started":"2024-02-21T01:21:25.793558Z","shell.execute_reply":"2024-02-21T01:21:25.806179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not submission:\n    # FOR TESTING SET READ_FILES TO TRUE\n    if TEST_MODE:\n        train = train.sample(500,random_state=42).reset_index(drop=True)\n        spectrograms = {}\n        for i,e in enumerate(train.spec_id.values):\n            if i%100==0: print(i,', ',end='')\n            x = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/{e}.parquet')\n            spectrograms[e] = x.values\n        all_eegs = {}\n        for i,e in enumerate(train.eeg_id.values):\n            if i%100==0: print(i,', ',end='')\n            x = np.load(f'/kaggle/input/eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n            all_eegs[e] = x\n        all_raw_eegs = {}\n        for i,e in enumerate(train.eeg_id.values):\n            if i%100==0: print(i,', ',end='')\n            x = eeg_from_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{e}.parquet')              \n            all_raw_eegs[e] = x\n    else:\n        spectrograms = None\n        all_eegs = None\n        all_raw_eegs = None\n        if DATA_TYPE=='both' or DATA_TYPE=='kaggle':\n            spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()\n        if DATA_TYPE=='both' or DATA_TYPE=='eeg':\n            all_eegs = np.load('/kaggle/input/eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()\n        if DATA_TYPE=='raw':\n            all_raw_eegs = np.load('/kaggle/input/brain-eegs/eegs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.808398Z","iopub.execute_input":"2024-02-21T01:21:25.808673Z","iopub.status.idle":"2024-02-21T01:21:25.824511Z","shell.execute_reply.started":"2024-02-21T01:21:25.808649Z","shell.execute_reply":"2024-02-21T01:21:25.823608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as albu\nfrom scipy.signal import butter, lfilter\n\nclass DataGenerator():\n    'Generates data for Keras'\n    def __init__(self, data, specs=None, eeg_specs=None, raw_eegs=None, augment=False, mode='train', data_type=DATA_TYPE): \n        self.data = data\n        self.augment = augment\n        self.mode = mode\n        self.data_type = data_type\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.raw_eegs = raw_eegs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, index):\n        X, y = self.data_generation(index)\n        if self.augment: X = self.augmentation(X)\n        return X, y\n    \n    def __call__(self):\n        for i in range(self.__len__()):\n            yield self.__getitem__(i)\n            \n            if i == self.__len__()-1:\n                self.on_epoch_end()\n                \n    def on_epoch_end(self):\n        if self.mode=='train': \n            self.data = self.data.sample(frac=1).reset_index(drop=True)\n    \n    def data_generation(self, index):\n        if self.data_type == 'both':\n            X,y = self.generate_all_specs(index)\n        elif self.data_type == 'eeg' or self.data_type == 'kaggle':\n            X,y = self.generate_specs(index)\n        elif self.data_type == 'raw':\n            X,y = self.generate_raw(index)\n\n        return X,y\n    \n    def generate_all_specs(self, index):\n        X = np.zeros((512,512,3),dtype='float32')\n        y = np.zeros((6,),dtype='float32')\n        \n        row = self.data.iloc[index]\n        if self.mode=='test': \n            offset = 0\n        else:\n            offset = int(row.offset/2)\n            \n        eeg = self.eeg_specs[row.eeg_id]\n        spec = self.specs[row.spec_id]\n        \n        imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in range(4)]\n        img = np.stack(imgs,axis=-1)\n        # LOG TRANSFORM SPECTROGRAM\n        img = np.clip(img,np.exp(-4),np.exp(8))\n        img = np.log(img)\n            \n        # STANDARDIZE PER IMAGE\n        img = np.nan_to_num(img, nan=0.0)    \n            \n        mn = img.flatten().min()\n        mx = img.flatten().max()\n        ep = 1e-5\n        img = 255 * (img - mn) / (mx - mn + ep)\n        X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n        X[100+56:200+56,:256,0] = img[:,22:-22,2]\n        X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n        X[100+56:200+56,:256,1] = img[:,22:-22,3]\n        \n        X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n        X[100+56:200+56,256:,0] = img[:,22:-22,1]\n        X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n        X[100+56:200+56,256:,1] = img[:,22:-22,3]\n        \n        # EEG\n        img = eeg\n        mn = img.flatten().min()\n        mx = img.flatten().max()\n        ep = 1e-5\n        img = 255 * (img - mn) / (mx - mn + ep)\n        X[200+56:300+56,:256,0] = img[:,22:-22,0]\n        X[300+56:400+56,:256,0] = img[:,22:-22,2]\n        X[200+56:300+56,:256,1] = img[:,22:-22,1]\n        X[300+56:400+56,:256,1] = img[:,22:-22,3]\n        \n        X[200+56:300+56,256:,0] = img[:,22:-22,0]\n        X[300+56:400+56,256:,0] = img[:,22:-22,1]\n        X[200+56:300+56,256:,1] = img[:,22:-22,2]\n        X[300+56:400+56,256:,1] = img[:,22:-22,3]\n        \n        if self.mode!='test':\n            y[:] = row[TARGETS]\n        \n        return X,y\n    \n    def generate_specs(self, index):\n        X = np.zeros((512,512,3),dtype='float32')\n        y = np.zeros((6,),dtype='float32')\n        \n        row = self.data.iloc[index]\n        if self.mode=='test': \n            offset = 0\n        else:\n            offset = int(row.offset/2)\n            \n        if self.data_type == 'eeg':\n            img = self.eeg_specs[row.eeg_id]\n        elif self.data_type == 'kaggle':\n            spec = self.specs[row.spec_id]\n            imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in range(4)]\n            img = np.stack(imgs,axis=-1)\n            # LOG TRANSFORM SPECTROGRAM\n            img = np.clip(img,np.exp(-4),np.exp(8))\n            img = np.log(img)\n            \n            # STANDARDIZE PER IMAGE\n            img = np.nan_to_num(img, nan=0.0)    \n            \n        mn = img.flatten().min()\n        mx = img.flatten().max()\n        ep = 1e-5\n        img = 255 * (img - mn) / (mx - mn + ep)\n        \n        X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n        X[100+56:200+56,:256,0] = img[:,22:-22,2]\n        X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n        X[100+56:200+56,:256,1] = img[:,22:-22,3]\n        \n        X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n        X[100+56:200+56,256:,0] = img[:,22:-22,1]\n        X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n        X[100+56:200+56,256:,1] = img[:,22:-22,3]\n        \n        X[200+56:300+56,:256,0] = img[:,22:-22,0]\n        X[300+56:400+56,:256,0] = img[:,22:-22,2]\n        X[200+56:300+56,:256,1] = img[:,22:-22,1]\n        X[300+56:400+56,:256,1] = img[:,22:-22,3]\n        \n        X[200+56:300+56,256:,0] = img[:,22:-22,0]\n        X[300+56:400+56,256:,0] = img[:,22:-22,1]\n        X[200+56:300+56,256:,1] = img[:,22:-22,2]\n        X[300+56:400+56,256:,1] = img[:,22:-22,3]\n        \n        if self.mode!='test':\n            y[:] = row[TARGETS]\n        \n        return X,y\n    \n    def generate_raw(self,index):\n        X = np.zeros((10_000,8),dtype='float32')\n        y = np.zeros((6,),dtype='float32')\n        \n        row = self.data.iloc[index]\n        eeg = self.raw_eegs[row.eeg_id]\n            \n        # FEATURE ENGINEER\n        X[:,0] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['T3']]\n        X[:,1] = eeg[:,FEAT2IDX['T3']] - eeg[:,FEAT2IDX['O1']]\n            \n        X[:,2] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['C3']]\n        X[:,3] = eeg[:,FEAT2IDX['C3']] - eeg[:,FEAT2IDX['O1']]\n            \n        X[:,4] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['C4']]\n        X[:,5] = eeg[:,FEAT2IDX['C4']] - eeg[:,FEAT2IDX['O2']]\n            \n        X[:,6] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['T4']]\n        X[:,7] = eeg[:,FEAT2IDX['T4']] - eeg[:,FEAT2IDX['O2']]\n            \n        # STANDARDIZE\n        X = np.clip(X,-1024,1024)\n        X = np.nan_to_num(X, nan=0) / 32.0\n            \n        # BUTTER LOW-PASS FILTER\n        X = self.butter_lowpass_filter(X)\n        # Downsample\n        X = X[::5,:]\n        \n        if self.mode!='test':\n            y[:] = row[TARGETS]\n                \n        return X,y\n        \n    def butter_lowpass_filter(self, data, cutoff_freq=20, sampling_rate=200, order=4):\n        nyquist = 0.5 * sampling_rate\n        normal_cutoff = cutoff_freq / nyquist\n        b, a = butter(order, normal_cutoff, btype='low', analog=False)\n        filtered_data = lfilter(b, a, data, axis=0)\n        return filtered_data\n    \n    def resize(self, img,size):\n        composition = albu.Compose([\n                albu.Resize(size[0],size[1])\n            ])\n        return composition(image=img)['image']\n            \n    def augmentation(self, img):\n        composition = albu.Compose([\n                albu.HorizontalFlip(p=0.4)\n            ])\n        return composition(image=img)['image']","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.826270Z","iopub.execute_input":"2024-02-21T01:21:25.826616Z","iopub.status.idle":"2024-02-21T01:21:25.879964Z","shell.execute_reply.started":"2024-02-21T01:21:25.826584Z","shell.execute_reply":"2024-02-21T01:21:25.878971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submission and DATA_TYPE!='raw':\n    gen = DataGenerator(train, augment=False, specs=spectrograms, eeg_specs=all_eegs, data_type=DATA_TYPE)\n    for x,y in gen:\n        break\n    plt.imshow(x[:,:,0])\n    plt.title(f'Target = {y.round(1)}',size=12)\n    plt.yticks([])\n    plt.ylabel('Frequencies (Hz)',size=12)\n    plt.xlabel('Time (sec)',size=12)\n    plt.show()\n    \nif not submission and DATA_TYPE=='raw':\n    gen = DataGenerator(train, raw_eegs=all_raw_eegs, data_type=DATA_TYPE)\n    for x,y in gen:\n        plt.figure(figsize=(20,4))\n        offset = 0\n        for j in range(x.shape[-1]):\n            if j!=0: offset -= x[:,j].min()\n            plt.plot(range(2_000),x[:,j]+offset,label=f'feature {j+1}')\n            offset += x[:,j].max()\n        plt.legend()\n        plt.show()\n        break","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.881172Z","iopub.execute_input":"2024-02-21T01:21:25.881455Z","iopub.status.idle":"2024-02-21T01:21:25.894891Z","shell.execute_reply.started":"2024-02-21T01:21:25.881431Z","shell.execute_reply":"2024-02-21T01:21:25.893956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submission:\n\n    def lrfn(epoch):\n        return [1e-3,1e-3,1e-4,1e-4][epoch]\n\n    LR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n    \n    def lrfn2(epoch):\n        return [1e-5,1e-5,1e-6][epoch]\n\n    LR2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.902090Z","iopub.execute_input":"2024-02-21T01:21:25.902380Z","iopub.status.idle":"2024-02-21T01:21:25.907976Z","shell.execute_reply.started":"2024-02-21T01:21:25.902355Z","shell.execute_reply":"2024-02-21T01:21:25.907055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate\n\ndef build_model():  \n    inp = tf.keras.layers.Input((512,512,3))\n    base_model = load_model(f'{LOAD_BACKBONE_FROM}')    \n    x = base_model(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    output = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n    model = tf.keras.Model(inputs=inp, outputs=output)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n    model.compile(loss=loss, optimizer=opt)  \n    return model\n\ndef score(y_true, y_pred):\n    kl = tf.keras.metrics.KLDivergence()\n    return kl(y_true, y_pred)\n\ndef wave_block(x, filters, kernel_size, n):\n    dilation_rates = [2**i for i in range(n)]\n    x = Conv1D(filters = filters,\n               kernel_size = 1,\n               padding = 'same')(x)\n    res_x = x\n    for dilation_rate in dilation_rates:\n        tanh_out = Conv1D(filters = filters,\n                          kernel_size = kernel_size,\n                          padding = 'same', \n                          activation = 'tanh', \n                          dilation_rate = dilation_rate)(x)\n        sigm_out = Conv1D(filters = filters,\n                          kernel_size = kernel_size,\n                          padding = 'same',\n                          activation = 'sigmoid', \n                          dilation_rate = dilation_rate)(x)\n        x = Multiply()([tanh_out, sigm_out])\n        x = Conv1D(filters = filters,\n                   kernel_size = 1,\n                   padding = 'same')(x)\n        res_x = Add()([res_x, x])\n    return res_x\n\ndef build_wave_model():\n        \n    # INPUT \n    inp = tf.keras.Input(shape=(2_000,8))\n    \n    ############\n    # FEATURE EXTRACTION SUB MODEL\n    inp2 = tf.keras.Input(shape=(2_000,1))\n    x = wave_block(inp2, 8, 3, 12)\n    x = wave_block(x, 16, 3, 8)\n    x = wave_block(x, 32, 3, 4)\n    x = wave_block(x, 64, 3, 1)\n    model2 = tf.keras.Model(inputs=inp2, outputs=x)\n    ###########\n    \n    # LEFT TEMPORAL CHAIN\n    x1 = model2(inp[:,:,0:1])\n    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n    x2 = model2(inp[:,:,1:2])\n    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n    z1 = tf.keras.layers.Average()([x1,x2])\n    \n    # LEFT PARASAGITTAL CHAIN\n    x1 = model2(inp[:,:,2:3])\n    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n    x2 = model2(inp[:,:,3:4])\n    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n    z2 = tf.keras.layers.Average()([x1,x2])\n    \n    # RIGHT PARASAGITTAL CHAIN\n    x1 = model2(inp[:,:,4:5])\n    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n    x2 = model2(inp[:,:,5:6])\n    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n    z3 = tf.keras.layers.Average()([x1,x2])\n    \n    # RIGHT TEMPORAL CHAIN\n    x1 = model2(inp[:,:,6:7])\n    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n    x2 = model2(inp[:,:,7:8])\n    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n    z4 = tf.keras.layers.Average()([x1,x2])\n    \n    # COMBINE CHAINS\n    y = tf.keras.layers.Concatenate()([z1,z2,z3,z4])\n    y = tf.keras.layers.Dense(64, activation='relu')(y)\n    y = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(y)\n    \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=y)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n    model.compile(loss=loss, optimizer = opt)\n    \n    return model\n\ndef plot_hist(hist):\n    metrics = ['loss']\n    for i,metric in enumerate(metrics):\n        plt.figure(figsize=(10,4))\n        plt.subplot(1,2,i+1)\n        plt.plot(hist[metric])\n        plt.plot(hist[f'val_{metric}'])\n        plt.title(f'{metric}',size=12)\n        plt.ylabel(f'{metric}',size=12)\n        plt.xlabel('epoch',size=12)\n        plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.909539Z","iopub.execute_input":"2024-02-21T01:21:25.909919Z","iopub.status.idle":"2024-02-21T01:21:25.937165Z","shell.execute_reply.started":"2024-02-21T01:21:25.909883Z","shell.execute_reply":"2024-02-21T01:21:25.936255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\nif not submission:\n    all_oof = []\n    all_true = []\n    losses = []\n    val_losses = []\n    total_hist = {}\n\n    gkf = GroupKFold(n_splits=5)\n    for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):   \n        \n        print('#'*25)\n        print(f'### Fold {i+1}')\n        \n        data, val = train.iloc[train_index],train.iloc[valid_index]\n        train_gen = DataGenerator(data, augment=False, specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n        valid_gen = DataGenerator(val, mode='valid', specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n        data, val = data[data['kl']<5.5],val[val['kl']<5.5]\n        train_gen2 = DataGenerator(data, augment=False, specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n        valid_gen2 = DataGenerator(val, mode='valid', specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n        in_shape = (2000,8) if DATA_TYPE=='raw' else (512,512,3)\n        EPOCHS = 4\n        BATCH_SIZE_PER_REPLICA = 32\n        BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n        train_dataset = tf.data.Dataset.from_generator(generator=train_gen, \n                                                   output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n                                                                     tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n        val_dataset = tf.data.Dataset.from_generator(generator=valid_gen, \n                                                   output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n                                                                     tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n        train_dataset2 = tf.data.Dataset.from_generator(generator=train_gen2, \n                                                   output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n                                                                     tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n        val_dataset2 = tf.data.Dataset.from_generator(generator=valid_gen2, \n                                                   output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n                                                                     tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n          \n        print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n        print('#'*25)\n        \n        K.clear_session()\n        with strategy.scope():\n            if DATA_TYPE=='raw':\n                model = build_wave_model()\n            else:\n                model = build_model()\n        \n        hist = model.fit(train_dataset, validation_data = val_dataset, \n                         epochs=EPOCHS, callbacks=[LR])\n        print(f'### seconds stage train size {len(data)}, valid size {len(val)}')\n        print('#'*25)\n        hist2 = model.fit(train_dataset2, validation_data = val_dataset, \n                         epochs=3, callbacks=[LR2])\n        losses.append(hist.history['loss']+hist2.history['loss'])\n        val_losses.append(hist.history['val_loss']+hist2.history['val_loss'])\n        K.clear_session()\n        with strategy.scope():\n            model.save_weights(f'model_{DATA_TYPE}_{VER}_{i}.weights.h5')\n        oof = model.predict(val_dataset, verbose=1)\n        all_oof.append(oof)\n        all_true.append(train.iloc[valid_index][TARGETS].values)    \n        del model, oof\n        gc.collect()\n        \n    total_hist['loss'] = np.mean(losses,axis=0)\n    total_hist['val_loss'] = np.mean(val_losses,axis=0)\n    all_oof = np.concatenate(all_oof)\n    all_true = np.concatenate(all_true)\n    plot_hist(total_hist)\n    print('#'*25)\n    print(f'CV KL SCORE: {score(all_true,all_oof)}')","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.938416Z","iopub.execute_input":"2024-02-21T01:21:25.938700Z","iopub.status.idle":"2024-02-21T01:21:25.959961Z","shell.execute_reply.started":"2024-02-21T01:21:25.938677Z","shell.execute_reply":"2024-02-21T01:21:25.959063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\nimport librosa\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((100,300,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n            # FILL NANS\n            x1 = eeg[COLS[kk]].values\n            x2 = eeg[COLS[kk+1]].values\n            m = np.nanmean(x1)\n            if np.isnan(x1).mean()<1: x1 = np.nan_to_num(x1,nan=m)\n            else: x1[:] = 0\n            m = np.nanmean(x2)\n            if np.isnan(x2).mean()<1: x2 = np.nan_to_num(x2,nan=m)\n            else: x2[:] = 0\n                \n            # COMPUTE PAIR DIFFERENCES\n            x = x1 - x2\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//300, \n                  n_fft=1024, n_mels=100, fmin=0, fmax=20, win_length=128)\n            \n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//30)*30\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.show()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.961354Z","iopub.execute_input":"2024-02-21T01:21:25.962054Z","iopub.status.idle":"2024-02-21T01:21:25.983174Z","shell.execute_reply.started":"2024-02-21T01:21:25.961980Z","shell.execute_reply":"2024-02-21T01:21:25.982164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if submission:\n    test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\n    print('Test shape',test.shape)\n    test.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:25.984177Z","iopub.execute_input":"2024-02-21T01:21:25.984444Z","iopub.status.idle":"2024-02-21T01:21:26.001073Z","shell.execute_reply.started":"2024-02-21T01:21:25.984422Z","shell.execute_reply":"2024-02-21T01:21:26.000066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nif submission:\n    PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\n    files2 = os.listdir(PATH2)\n    print(f'There are {len(files2)} test spectrogram parquets')\n    \n    spectrograms2 = {}\n    for i,f in enumerate(files2):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH2}{f}')\n        name = int(f.split('.')[0])\n        spectrograms2[name] = tmp.iloc[:,1:].values\n    \n    # RENAME FOR DATA GENERATOR\n    test = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:26.002695Z","iopub.execute_input":"2024-02-21T01:21:26.003360Z","iopub.status.idle":"2024-02-21T01:21:26.042933Z","shell.execute_reply.started":"2024-02-21T01:21:26.003318Z","shell.execute_reply":"2024-02-21T01:21:26.041891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nif submission:\n    PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\n    DISPLAY = 0\n    EEG_IDS2 = test.eeg_id.unique()\n    all_eegs2 = {}\n\n    print('Converting Test EEG to Spectrograms...'); print()\n    for i,eeg_id in enumerate(EEG_IDS2):\n        \n        # CREATE SPECTROGRAM FROM EEG PARQUET\n        img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n        all_eegs2[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:26.044414Z","iopub.execute_input":"2024-02-21T01:21:26.045160Z","iopub.status.idle":"2024-02-21T01:21:26.306810Z","shell.execute_reply.started":"2024-02-21T01:21:26.045121Z","shell.execute_reply":"2024-02-21T01:21:26.305337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL RAW EEG SIGNALS\nif submission :\n    all_raw_eegs2 = {}\n    EEG_IDS2 = test.eeg_id.unique()\n    PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\n\n    print('Processing Test EEG parquets...'); print()\n    for i,eeg_id in enumerate(EEG_IDS2):\n        \n        # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n        data = eeg_from_parquet(f'{PATH2}{eeg_id}.parquet')\n        all_raw_eegs2[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:26.308443Z","iopub.execute_input":"2024-02-21T01:21:26.309784Z","iopub.status.idle":"2024-02-21T01:21:26.332685Z","shell.execute_reply.started":"2024-02-21T01:21:26.309742Z","shell.execute_reply":"2024-02-21T01:21:26.331385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission ON TEST without ensemble\nif submission and not ENSEMBLE:\n    preds = []\n    \n    if DATA_TYPE=='raw':\n        test_gen = DataGenerator(test, mode='test', raw_eegs=all_raw_eegs2)\n        in_shape = (2000,8)\n    else:\n        test_gen = DataGenerator(test, mode='test', specs = spectrograms2, eeg_specs = all_eegs2)\n        in_shape = (512,512,3)\n    \n    test_dataset = tf.data.Dataset.from_generator(generator=test_gen, \n                                               output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n                                                                 tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(64).prefetch(tf.data.AUTOTUNE)\n    if DATA_TYPE=='raw':\n        model = build_wave_model()\n    else:\n        model = build_model()\n\n    for i in range(5):\n        print(f'Fold {i+1}')\n        model.load_weights(f'{LOAD_MODELS_FROM}model_{DATA_TYPE}_{VER}_{i}.weights.h5')\n        pred = model.predict(test_dataset, verbose=1)\n        preds.append(pred)\n        \n    pred = np.mean(preds,axis=0)\n    print('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:26.334459Z","iopub.execute_input":"2024-02-21T01:21:26.335656Z","iopub.status.idle":"2024-02-21T01:21:26.352392Z","shell.execute_reply.started":"2024-02-21T01:21:26.335603Z","shell.execute_reply":"2024-02-21T01:21:26.351098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission ON TEST with ensemble\nif submission and ENSEMBLE:\n    preds = []\n    test_gen_kaggle = DataGenerator(test, mode='test', data_type='kaggle', specs = spectrograms2, eeg_specs = all_eegs2)\n    test_dataset_kaggle = tf.data.Dataset.from_generator(generator=test_gen_kaggle, \n                                               output_signature=(tf.TensorSpec(shape=(512,512,3), dtype=tf.float32),\n                                                                 tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(64).prefetch(tf.data.AUTOTUNE)\n    test_gen_both = DataGenerator(test, mode='test', data_type='both', specs = spectrograms2, eeg_specs = all_eegs2)\n    test_dataset_both = tf.data.Dataset.from_generator(generator=test_gen_both, \n                                               output_signature=(tf.TensorSpec(shape=(512,512,3), dtype=tf.float32),\n                                                                 tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(64).prefetch(tf.data.AUTOTUNE)\n\n    test_gen_eeg = DataGenerator(test, mode='test', data_type='eeg', specs = spectrograms2, eeg_specs = all_eegs2)\n    test_dataset_eeg = tf.data.Dataset.from_generator(generator=test_gen_eeg, \n                                               output_signature=(tf.TensorSpec(shape=(512,512,3), dtype=tf.float32),\n                                                                 tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(64).prefetch(tf.data.AUTOTUNE)\n    test_gen_raw = DataGenerator(test, mode='test', data_type='raw', raw_eegs=all_raw_eegs2)\n    test_dataset_raw = tf.data.Dataset.from_generator(generator=test_gen_raw, \n                                               output_signature=(tf.TensorSpec(shape=(2000,8), dtype=tf.float32),\n                                                                 tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(64).prefetch(tf.data.AUTOTUNE)\n \n    # LB SCORE FOR EACH MODEL\n    lbs = 1 - np.array(LBs)\n    weights = lbs/lbs.sum()\n    model = build_model()\n    model_wave = build_wave_model()\n\n    for i in range(5):\n        print(f'Fold {i+1}')\n        \n        model.load_weights(f'{LOAD_MODELS_FROM}model_kaggle_{VERK}_{i}.weights.h5')\n        pred_kaggle = model.predict(test_dataset_kaggle, verbose=1)\n        \n        model.load_weights(f'{LOAD_MODELS_FROM}model_both_{VERB}_{i}.weights.h5')\n        pred_both = model.predict(test_dataset_both, verbose=1)\n        \n        model.load_weights(f'{LOAD_MODELS_FROM}model_eeg_{VERE}_{i}.weights.h5')\n        pred_eeg = model.predict(test_dataset_eeg, verbose=1)\n        \n        model_wave.load_weights(f'{LOAD_MODELS_FROM}model_raw_{VERR}_{i}.weights.h5')\n        pred_raw = model_wave.predict(test_dataset_raw, verbose=1)\n        \n        pred = np.array([pred_both,pred_eeg,pred_kaggle,pred_raw])\n        pred = np.average(pred,axis=0,weights=weights)\n        preds.append(pred)\n        \n    pred = np.mean(preds,axis=0)\n    print('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:21:26.354618Z","iopub.execute_input":"2024-02-21T01:21:26.355585Z","iopub.status.idle":"2024-02-21T01:22:31.554884Z","shell.execute_reply.started":"2024-02-21T01:21:26.355534Z","shell.execute_reply":"2024-02-21T01:22:31.553932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\nlabels=['seizure','lpd','gpd','lrda','grda','other']\nfor i in range(len(labels)):\n    submission[f'{labels[i]}_vote']=(preds_combine[:, i]*0.4 + pred[:, i] * 0.6)\n#     submission[f'{labels[i]}_vote']=(test_preds[:,i]*0.1 + preds_combine[:, i]*0.18 + test_predss[:, i]*0.16 + pred[:, i] * 0.56)\nsubmission.to_csv(\"submission.csv\",index=None)\ndisplay(submission.head())","metadata":{"execution":{"iopub.status.busy":"2024-02-21T02:59:00.005296Z","iopub.execute_input":"2024-02-21T02:59:00.005669Z","iopub.status.idle":"2024-02-21T02:59:00.343318Z","shell.execute_reply.started":"2024-02-21T02:59:00.005642Z","shell.execute_reply":"2024-02-21T02:59:00.342137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsubmission.iloc[:,-6:].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T01:22:31.579779Z","iopub.execute_input":"2024-02-21T01:22:31.580082Z","iopub.status.idle":"2024-02-21T01:22:31.588052Z","shell.execute_reply.started":"2024-02-21T01:22:31.580058Z","shell.execute_reply":"2024-02-21T01:22:31.587059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}