{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7403069,"sourceType":"datasetVersion","datasetId":4304949},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":158958765,"sourceType":"kernelVersion"},{"sourceId":235484897,"sourceType":"kernelVersion"},{"sourceId":242175468,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n\nVER = 5\n\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = '/kaggle/input/brain-efficientnet-models-v3-v4-v5/'\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:55:56.941043Z","iopub.execute_input":"2025-05-30T07:55:56.941615Z","iopub.status.idle":"2025-05-30T07:56:09.602696Z","shell.execute_reply.started":"2025-05-30T07:55:56.941589Z","shell.execute_reply":"2025-05-30T07:56:09.601826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:56:59.456056Z","iopub.execute_input":"2025-05-30T07:56:59.456750Z","iopub.status.idle":"2025-05-30T07:56:59.461923Z","shell.execute_reply.started":"2025-05-30T07:56:59.456720Z","shell.execute_reply":"2025-05-30T07:56:59.461029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:57:01.815695Z","iopub.execute_input":"2025-05-30T07:57:01.816318Z","iopub.status.idle":"2025-05-30T07:57:01.821077Z","shell.execute_reply.started":"2025-05-30T07:57:01.816296Z","shell.execute_reply":"2025-05-30T07:57:01.820185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:57:03.995697Z","iopub.execute_input":"2025-05-30T07:57:03.996324Z","iopub.status.idle":"2025-05-30T07:57:04.237092Z","shell.execute_reply.started":"2025-05-30T07:57:03.996298Z","shell.execute_reply":"2025-05-30T07:57:04.236419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:57:06.376439Z","iopub.execute_input":"2025-05-30T07:57:06.377266Z","iopub.status.idle":"2025-05-30T07:57:06.465993Z","shell.execute_reply.started":"2025-05-30T07:57:06.377234Z","shell.execute_reply":"2025-05-30T07:57:06.465154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nREAD_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:57:29.945614Z","iopub.execute_input":"2025-05-30T07:57:29.946250Z","iopub.status.idle":"2025-05-30T07:58:14.261027Z","shell.execute_reply.started":"2025-05-30T07:57:29.946224Z","shell.execute_reply":"2025-05-30T07:58:14.260255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %%time\n# READ_SPEC_FILES = False\n\n# # READ ALL SPECTROGRAMS\n# PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\n# files = os.listdir(PATH)\n# print(f'There are {len(files)} spectrogram parquets')\n\n# if READ_SPEC_FILES:    \n#     spectrograms = {}\n#     for i,f in enumerate(files):\n#         if i%100==0: print(i,', ',end='')\n#         tmp = pd.read_parquet(f'{PATH}{f}')\n#         name = int(f.split('.')[0])\n#         spectrograms[name] = tmp.iloc[:,1:].values\n# else:\n#     spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:37:41.060328Z","iopub.execute_input":"2025-05-30T07:37:41.060621Z","iopub.status.idle":"2025-05-30T07:38:22.709054Z","shell.execute_reply.started":"2025-05-30T07:37:41.060602Z","shell.execute_reply":"2025-05-30T07:38:22.708344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nREAD_EEG_SPEC_FILES = True\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i%100==0: print(i,', ',end='')\n        x = np.load(f'/kaggle/input/preprocessing/preprocessed/spec/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:59:02.795979Z","iopub.execute_input":"2025-05-30T07:59:02.796677Z","iopub.status.idle":"2025-05-30T08:01:09.651716Z","shell.execute_reply.started":"2025-05-30T07:59:02.796650Z","shell.execute_reply":"2025-05-30T08:01:09.650849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %%time\n# READ_EEG_SPEC_FILES = False\n\n# if READ_EEG_SPEC_FILES:\n#     all_eegs = {}\n#     for i,e in enumerate(train.eeg_id.values):\n#         if i%100==0: print(i,', ',end='')\n#         x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n#         all_eegs[e] = x\n# else:\n#     all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:38:22.710254Z","iopub.execute_input":"2025-05-30T07:38:22.710454Z","iopub.status.idle":"2025-05-30T07:39:13.267353Z","shell.execute_reply.started":"2025-05-30T07:38:22.710438Z","shell.execute_reply":"2025-05-30T07:39:13.266604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import *\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:02:47.389791Z","iopub.execute_input":"2025-05-30T08:02:47.390464Z","iopub.status.idle":"2025-05-30T08:02:47.577578Z","shell.execute_reply.started":"2025-05-30T08:02:47.390432Z","shell.execute_reply":"2025-05-30T08:02:47.576977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import albumentations as albu\n# TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\n# TARS2 = {x:y for y,x in TARS.items()}\n\n# class DataGenerator(tf.keras.utils.Sequence):\n#     'Generates data for Keras'\n#     def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n#                  specs = spectrograms, eeg_specs = all_eegs): \n\n#         self.data = data\n#         self.batch_size = batch_size\n#         self.shuffle = shuffle\n#         self.augment = augment\n#         self.mode = mode\n#         self.specs = specs\n#         self.eeg_specs = eeg_specs\n#         self.on_epoch_end()\n        \n#     def __len__(self):\n#         'Denotes the number of batches per epoch'\n#         ct = int( np.ceil( len(self.data) / self.batch_size ) )\n#         return ct\n\n#     def __getitem__(self, index):\n#         'Generate one batch of data'\n#         indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n#         X, y = self.__data_generation(indexes)\n#         if self.augment: X = self.__augment_batch(X) \n#         return X, y\n\n#     def on_epoch_end(self):\n#         'Updates indexes after each epoch'\n#         self.indexes = np.arange( len(self.data) )\n#         if self.shuffle: np.random.shuffle(self.indexes)\n                        \n#     def __data_generation(self, indexes):\n#         'Generates data containing batch_size samples' \n        \n#         X = np.zeros((len(indexes),128,256,8),dtype='float32')\n#         y = np.zeros((len(indexes),6),dtype='float32')\n#         img = np.ones((128,256),dtype='float32')\n        \n#         for j,i in enumerate(indexes):\n#             row = self.data.iloc[i]\n#             if self.mode=='test': \n#                 r = 0\n#             else: \n#                 r = int( (row['min'] + row['max'])//4 )\n\n#             for k in range(4):\n#                 # EXTRACT 300 ROWS OF SPECTROGRAM\n#                 img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n#                 # LOG TRANSFORM SPECTROGRAM\n#                 img = np.clip(img,np.exp(-4),np.exp(8))\n#                 img = np.log(img)\n                \n#                 # STANDARDIZE PER IMAGE\n#                 ep = 1e-6\n#                 m = np.nanmean(img.flatten())\n#                 s = np.nanstd(img.flatten())\n#                 img = (img-m)/(s+ep)\n#                 img = np.nan_to_num(img, nan=0.0)\n                \n#                 # CROP TO 256 TIME STEPS\n#                 X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n#             # EEG SPECTROGRAMS\n#             img = self.eeg_specs[row.eeg_id]\n#             X[j,:,:,4:] = img\n                \n#             if self.mode!='test':\n#                 y[j,] = row[TARGETS]\n            \n#         return X,y\n    \n#     def __random_transform(self, img):\n#         composition = albu.Compose([\n#             albu.HorizontalFlip(p=0.5),\n#             #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n#         ])\n#         return composition(image=img)['image']\n            \n#     def __augment_batch(self, img_batch):\n#         for i in range(img_batch.shape[0]):\n#             img_batch[i, ] = self.__random_transform(img_batch[i, ])\n#         return img_batch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T07:39:49.299860Z","iopub.execute_input":"2025-05-30T07:39:49.300165Z","iopub.status.idle":"2025-05-30T07:39:49.323517Z","shell.execute_reply.started":"2025-05-30T07:39:49.300138Z","shell.execute_reply":"2025-05-30T07:39:49.322293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as albu\nimport cv2  # Required for resizing\n\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs=spectrograms, eeg_specs=all_eegs): \n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        return int(np.ceil(len(self.data) / self.batch_size))\n\n    def __getitem__(self, index):\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment:\n            X = self.__augment_batch(X)\n        return X, y\n\n    def on_epoch_end(self):\n        self.indexes = np.arange(len(self.data))\n        if self.shuffle:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, indexes):\n        X = np.zeros((len(indexes), 128, 256, 8), dtype='float32')\n        y = np.zeros((len(indexes), 6), dtype='float32')\n        \n        for j, i in enumerate(indexes):\n            row = self.data.iloc[i]\n            r = 0 if self.mode == 'test' else int((row['min'] + row['max']) // 4)\n\n            for k in range(4):\n                img = self.specs[row.spec_id][r:r+300, k*100:(k+1)*100].T\n                img = np.clip(img, np.exp(-4), np.exp(8))\n                img = np.log(img)\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img - m) / (s + 1e-6)\n                img = np.nan_to_num(img, nan=0.0)\n                X[j, 14:-14, :, k] = img[:, 22:-22] / 2.0\n\n            # EEG SPECTROGRAMS (with resizing to (128, 256, 4))\n            img = self.eeg_specs[row.eeg_id]\n\n            if img.shape != (128, 256, 4):\n                resized_img = np.zeros((128, 256, 4), dtype='float32')\n                for ch in range(min(4, img.shape[-1])):\n                    resized_img[:, :, ch] = cv2.resize(img[:, :, ch], (256, 128), interpolation=cv2.INTER_LINEAR)\n                img = resized_img\n\n            X[j, :, :, 4:] = img\n\n            if self.mode != 'test':\n                y[j,] = row[TARGETS]\n\n        return X, y\n\n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            # albu.CoarseDropout(...) can be re-enabled here if needed\n        ])\n        return composition(image=img)['image']\n\n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:02:50.914790Z","iopub.execute_input":"2025-05-30T08:02:50.915140Z","iopub.status.idle":"2025-05-30T08:02:55.318737Z","shell.execute_reply.started":"2025-05-30T08:02:50.915099Z","shell.execute_reply":"2025-05-30T08:02:55.317930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gen = DataGenerator(train, batch_size=32, shuffle=False)\nROWS=2; COLS=3; BATCHES=2\n\nfor i,(x,y) in enumerate(gen):\n    plt.figure(figsize=(20,8))\n    for j in range(ROWS):\n        for k in range(COLS):\n            plt.subplot(ROWS,COLS,j*COLS+k+1)\n            t = y[j*COLS+k]\n            img = x[j*COLS+k,:,:,0][::-1,]\n            mn = img.flatten().min()\n            mx = img.flatten().max()\n            img = (img-mn)/(mx-mn)\n            plt.imshow(img)\n            tars = f'[{t[0]:0.2f}'\n            for s in t[1:]: tars += f', {s:0.2f}'\n            eeg = train.eeg_id.values[i*32+j*COLS+k]\n            plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n            plt.yticks([])\n            plt.ylabel('Frequencies (Hz)',size=14)\n            plt.xlabel('Time (sec)',size=16)\n    plt.show()\n    if i==BATCHES-1: break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:02:59.950223Z","iopub.execute_input":"2025-05-30T08:02:59.950887Z","iopub.status.idle":"2025-05-30T08:03:02.192936Z","shell.execute_reply.started":"2025-05-30T08:02:59.950864Z","shell.execute_reply":"2025-05-30T08:03:02.191875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\nLR_START = 1e-6\nLR_MAX = 1e-3\nLR_MIN = 1e-6\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS2 = 10\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS2)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y, '-o')\nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Cosine Training Schedule',size=16); plt.show()\n\nLR2 = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:03:14.510140Z","iopub.execute_input":"2025-05-30T08:03:14.510874Z","iopub.status.idle":"2025-05-30T08:03:14.672803Z","shell.execute_reply.started":"2025-05-30T08:03:14.510850Z","shell.execute_reply":"2025-05-30T08:03:14.672031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LR_START = 1e-4\nLR_MAX = 1e-3\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.1\nEVERY = 1\nEPOCHS = 4\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//EVERY)\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, y, 'o-'); \nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Step Training Schedule',size=16); plt.show()\n\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:03:16.829407Z","iopub.execute_input":"2025-05-30T08:03:16.829707Z","iopub.status.idle":"2025-05-30T08:03:16.987863Z","shell.execute_reply.started":"2025-05-30T08:03:16.829685Z","shell.execute_reply":"2025-05-30T08:03:16.987132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:03:19.934671Z","iopub.execute_input":"2025-05-30T08:03:19.935318Z","iopub.status.idle":"2025-05-30T08:03:24.309022Z","shell.execute_reply.started":"2025-05-30T08:03:19.935294Z","shell.execute_reply":"2025-05-30T08:03:24.307968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\ndef build_model():\n    \n    inp = tf.keras.Input(shape=(128,256,8))\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n    \n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n    # EEG SPECTROGRAMS\n    x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = tf.keras.layers.Concatenate(axis=2)([x1,x2])\n    elif USE_EEG_SPECTROGRAMS: x = x2\n    else: x = x1\n    x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt) \n        \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:03:31.790887Z","iopub.execute_input":"2025-05-30T08:03:31.791750Z","iopub.status.idle":"2025-05-30T08:03:31.856373Z","shell.execute_reply.started":"2025-05-30T08:03:31.791717Z","shell.execute_reply":"2025-05-30T08:03:31.855551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\nall_oof = []\nall_true = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=False)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'EffNet_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n        \n    oof = model.predict(valid_gen, verbose=1)\n    all_oof.append(oof)\n    all_true.append(train.iloc[valid_index][TARGETS].values)\n    \n    del model, oof\n    gc.collect()\n    \nall_oof = np.concatenate(all_oof)\nall_true = np.concatenate(all_true)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:03:34.750012Z","iopub.execute_input":"2025-05-30T08:03:34.751660Z","iopub.status.idle":"2025-05-30T08:06:33.429525Z","shell.execute_reply.started":"2025-05-30T08:03:34.751622Z","shell.execute_reply":"2025-05-30T08:06:33.428649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -l /kaggle/input/kaggle-kl-div/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:06:48.275776Z","iopub.execute_input":"2025-05-30T08:06:48.276077Z","iopub.status.idle":"2025-05-30T08:06:48.651478Z","shell.execute_reply.started":"2025-05-30T08:06:48.276060Z","shell.execute_reply":"2025-05-30T08:06:48.650471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import importlib.util\nimport sys\n\n# Import kaggle_metric_utilities\nutil_path = '/kaggle/input/kaggle-kl-div/kaggle_metric_utilities.py'\nutil_spec = importlib.util.spec_from_file_location(\"kaggle_metric_utilities\", util_path)\nkaggle_metric_utilities = importlib.util.module_from_spec(util_spec)\nsys.modules[\"kaggle_metric_utilities\"] = kaggle_metric_utilities\nutil_spec.loader.exec_module(kaggle_metric_utilities)\n\n# Now import kaggle_kl_div (after utility is loaded)\nkl_path = '/kaggle/input/kaggle-kl-div/kaggle_kl_div.py'\nkl_spec = importlib.util.spec_from_file_location(\"kaggle_kl_div\", kl_path)\nkaggle_kl_div = importlib.util.module_from_spec(kl_spec)\nsys.modules[\"kaggle_kl_div\"] = kaggle_kl_div\nkl_spec.loader.exec_module(kaggle_kl_div)\n\n# Grab the score function\nscore = kaggle_kl_div.score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:07:01.230479Z","iopub.execute_input":"2025-05-30T08:07:01.230779Z","iopub.status.idle":"2025-05-30T08:07:01.239471Z","shell.execute_reply.started":"2025-05-30T08:07:01.230756Z","shell.execute_reply":"2025-05-30T08:07:01.238756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oof = pd.DataFrame(all_oof.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score KL-Div for EfficientNetB2 =', cv)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:07:03.869731Z","iopub.execute_input":"2025-05-30T08:07:03.870011Z","iopub.status.idle":"2025-05-30T08:07:03.938249Z","shell.execute_reply.started":"2025-05-30T08:07:03.869993Z","shell.execute_reply":"2025-05-30T08:07:03.937438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import importlib.util\nimport sys\nimport numpy as np\nimport pandas as pd\n\n# Load kaggle_metric_utilities\nutil_path = '/kaggle/input/kaggle-kl-div/kaggle_metric_utilities.py'\nutil_spec = importlib.util.spec_from_file_location(\"kaggle_metric_utilities\", util_path)\nkaggle_metric_utilities = importlib.util.module_from_spec(util_spec)\nsys.modules[\"kaggle_metric_utilities\"] = kaggle_metric_utilities\nutil_spec.loader.exec_module(kaggle_metric_utilities)\n\n# Load kaggle_kl_div\nkl_path = '/kaggle/input/kaggle-kl-div/kaggle_kl_div.py'\nkl_spec = importlib.util.spec_from_file_location(\"kaggle_kl_div\", kl_path)\nkaggle_kl_div = importlib.util.module_from_spec(kl_spec)\nsys.modules[\"kaggle_kl_div\"] = kaggle_kl_div\nkl_spec.loader.exec_module(kaggle_kl_div)\n\n# Grab the score function\nscore = kaggle_kl_div.score\n\n# Wrap predictions into DataFrames\noof = pd.DataFrame(all_oof.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\n# KL-Divergence Score\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score KL-Div for EfficientNetB2 =', cv)\n\n# Accuracy\npred_labels = np.argmax(all_oof, axis=1)\ntrue_labels = np.argmax(all_true, axis=1)\naccuracy = np.mean(pred_labels == true_labels)\nprint('Accuracy =', accuracy)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T08:07:42.086694Z","iopub.execute_input":"2025-05-30T08:07:42.087326Z","iopub.status.idle":"2025-05-30T08:07:42.140068Z","shell.execute_reply.started":"2025-05-30T08:07:42.087301Z","shell.execute_reply":"2025-05-30T08:07:42.139351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}