{"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":1493925,"sourceType":"datasetVersion","datasetId":877193},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":8032286,"sourceType":"datasetVersion","datasetId":4734573},{"sourceId":8048333,"sourceType":"datasetVersion","datasetId":4746032}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":270.012179,"end_time":"2024-01-14T22:56:02.916427","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-14T22:51:32.904248","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\nVER = 1\n\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = None\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = False","metadata":{"papermill":{"duration":14.80928,"end_time":"2024-01-14T22:51:51.64702","exception":false,"start_time":"2024-01-14T22:51:36.83774","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T07:10:06.63014Z","iopub.execute_input":"2024-04-30T07:10:06.630734Z","iopub.status.idle":"2024-04-30T07:10:06.640573Z","shell.execute_reply.started":"2024-04-30T07:10:06.630703Z","shell.execute_reply":"2024-04-30T07:10:06.63963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"papermill":{"duration":0.016556,"end_time":"2024-01-14T22:51:51.671783","exception":false,"start_time":"2024-01-14T22:51:51.655227","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:09:01.656429Z","iopub.execute_input":"2024-04-30T05:09:01.656965Z","iopub.status.idle":"2024-04-30T05:09:01.662847Z","shell.execute_reply.started":"2024-04-30T05:09:01.656936Z","shell.execute_reply":"2024-04-30T05:09:01.661882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class paths:\n    OUTPUT_DIR = \"/kaggle/working/\"\n    BASE_PATH = \"/kaggle/input/hms-harmful-brain-activity-classification\"\n    TRAIN_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\"\n    TRAIN_EEGS = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs\"\n    TRAIN_SPECTOGRAMS = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms\"\n    \n    TEST_CSV = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\n    TEST_EEGS = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs\"\n    TEST_SPECTOGRAMS = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms\"\n    \n    SAMPLE_SPECTOGRAM = \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/1000086677.parquet\"\n    SAMPLE_EEG = \"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet\"\n    \n    NPY_EEGS = \"/kaggle/input/hms-eeg-cache/eegs.npy\"\n    NPY_SPECTOGRAMS = \"/kaggle/input/hms-spectogram-cache/spectrogram.npy\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:09:01.664137Z","iopub.execute_input":"2024-04-30T05:09:01.664443Z","iopub.status.idle":"2024-04-30T05:09:01.681287Z","shell.execute_reply.started":"2024-04-30T05:09:01.66441Z","shell.execute_reply":"2024-04-30T05:09:01.680484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Data","metadata":{"papermill":{"duration":0.007846,"end_time":"2024-01-14T22:51:51.688268","exception":false,"start_time":"2024-01-14T22:51:51.680422","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df = pd.read_csv(paths.TRAIN_CSV)\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.288611,"end_time":"2024-01-14T22:51:51.984993","exception":false,"start_time":"2024-01-14T22:51:51.696382","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:09:01.682658Z","iopub.execute_input":"2024-04-30T05:09:01.682969Z","iopub.status.idle":"2024-04-30T05:09:01.984684Z","shell.execute_reply.started":"2024-04-30T05:09:01.682942Z","shell.execute_reply":"2024-04-30T05:09:01.983718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Non-Overlapping Eeg Id Train Data\nThe competition data description says that test data does not have multiple crops from the same `eeg_id`. Therefore we will train and validate using only 1 crop per `eeg_id`. There is a discussion about this [here][1].\n\n[1]: https://www.kaggle.com/competitions/hms-harmful-brain-activity-classification/discussion/467021","metadata":{"papermill":{"duration":0.009407,"end_time":"2024-01-14T22:51:52.004075","exception":false,"start_time":"2024-01-14T22:51:51.994668","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"papermill":{"duration":0.111621,"end_time":"2024-01-14T22:51:52.125134","exception":false,"start_time":"2024-01-14T22:51:52.013513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:09:01.987125Z","iopub.execute_input":"2024-04-30T05:09:01.987469Z","iopub.status.idle":"2024-04-30T05:09:02.065911Z","shell.execute_reply.started":"2024-04-30T05:09:01.987442Z","shell.execute_reply":"2024-04-30T05:09:02.064994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Train Spectrograms \n\nFirst we need to read in all 11k train spectrogram files. Reading thousands of files takes 11 minutes with Pandas. Instead, we can read 1 file from my [Kaggle dataset here][1] which contains all the 11k spectrograms in less than 1 minute! To use my Kaggle dataset, set variable `READ_SPEC_FILES = False`. Thank you for upvoting my helpful [dataset][1] :-)\n\n[1]: https://www.kaggle.com/datasets/cdeotte/brain-spectrograms","metadata":{"papermill":{"duration":0.00881,"end_time":"2024-01-14T22:51:52.142747","exception":false,"start_time":"2024-01-14T22:51:52.133937","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nREAD_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = paths.TRAIN_SPECTOGRAMS + '/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load(paths.NPY_SPECTOGRAMS,allow_pickle=True).item()","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"papermill":{"duration":55.16894,"end_time":"2024-01-14T22:52:47.320438","exception":false,"start_time":"2024-01-14T22:51:52.151498","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:09:02.067119Z","iopub.execute_input":"2024-04-30T05:09:02.067482Z","iopub.status.idle":"2024-04-30T05:10:05.455608Z","shell.execute_reply.started":"2024-04-30T05:09:02.067448Z","shell.execute_reply":"2024-04-30T05:10:05.454569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read EEG Spectrograms\nIn version 4 onward, we use EEG spectrograms in addition to Kaggle spectrograms. The EEG spectrograms come from my Kaggle dataset [here][4] (which were created from my spectrogram starter [here][5]). Thank you for upvoting my Kaggle dataset!\n\n[4]: https://www.kaggle.com/datasets/cdeotte/brain-eeg-spectrograms\n[5]: https://www.kaggle.com/code/cdeotte/how-to-make-spectrogram-from-eeg","metadata":{}},{"cell_type":"code","source":"%%time\nREAD_EEG_SPEC_FILES = False\n\n# READ ALL EEG SPECTROGRAMS\nPATH = paths.TRAIN_EEGS + '/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} EEG parquets')\n\nif READ_EEG_SPEC_FILES:\n    eegs = {}\n    for i,f in enumerate(files):\n        if i%100==0: print(i,', ',end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        eegs[name] = tmp.iloc[:,1:].values\nelse:\n    eegs = np.load(paths.NPY_EEGS,allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:10:05.456672Z","iopub.execute_input":"2024-04-30T05:10:05.456924Z","iopub.status.idle":"2024-04-30T05:11:27.031482Z","shell.execute_reply.started":"2024-04-30T05:10:05.456901Z","shell.execute_reply":"2024-04-30T05:11:27.030499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train DataLoader\nThis dataloader outputs 4 spectrogram images as a 4 channel image of size 128x256x4 per train sample. This notebook version is not using data augmention but the code is available below to experiment with albumentations data augmention. Just add `augment = True` when creating the train data loader. And consider adding new transformations to the augment function below.\n\nUPDATE: In version 4 onward, our dataloader outputs both Kaggle spectrograms and EEG spectrogams as 8 channel image of size 128x256x8. And we use horizonal data augmentation.","metadata":{"papermill":{"duration":0.010384,"end_time":"2024-01-14T22:52:47.341581","exception":false,"start_time":"2024-01-14T22:52:47.331197","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import albumentations as albu\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs = spectrograms, eeg_specs = eegs): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = False\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X) \n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test': \n                r = 0\n            else: \n                r = int( (row['min'] + row['max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n                \n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n            \n        return X,y\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch","metadata":{"papermill":{"duration":2.369789,"end_time":"2024-01-14T22:52:49.721728","exception":false,"start_time":"2024-01-14T22:52:47.351939","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:11:27.032895Z","iopub.execute_input":"2024-04-30T05:11:27.033203Z","iopub.status.idle":"2024-04-30T05:11:29.064994Z","shell.execute_reply.started":"2024-04-30T05:11:27.033176Z","shell.execute_reply":"2024-04-30T05:11:29.064119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display DataLoader\nBelow we display example dataloader spectrogram images.","metadata":{"papermill":{"duration":0.00888,"end_time":"2024-01-14T22:52:49.739973","exception":false,"start_time":"2024-01-14T22:52:49.731093","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gen = DataGenerator(train, batch_size=32, shuffle=False)\nROWS=2; COLS=3; BATCHES=1\n\nfor i,(x,y) in enumerate(gen):\n    plt.figure(figsize=(20,8))\n    for j in range(ROWS):\n        for k in range(COLS):\n            plt.subplot(ROWS,COLS,j*COLS+k+1)\n            t = y[j*COLS+k]\n            img = x[j*COLS+k,:,:,0][::-1,]\n            mn = img.flatten().min()\n            mx = img.flatten().max()\n            img = (img-mn)/(mx-mn)\n            plt.imshow(img)\n            tars = f'[{t[0]:0.2f}'\n            for s in t[1:]: tars += f', {s:0.2f}'\n            eeg = train.eeg_id.values[i*32+j*COLS+k]\n            plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n            plt.yticks([])\n            plt.ylabel('Frequencies (Hz)',size=14)\n            plt.xlabel('Time (sec)',size=16)\n    plt.show()\n    if i==BATCHES-1: break","metadata":{"papermill":{"duration":2.448242,"end_time":"2024-01-14T22:52:52.197249","exception":false,"start_time":"2024-01-14T22:52:49.749007","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:11:29.066365Z","iopub.execute_input":"2024-04-30T05:11:29.066894Z","iopub.status.idle":"2024-04-30T05:11:30.745419Z","shell.execute_reply.started":"2024-04-30T05:11:29.066864Z","shell.execute_reply":"2024-04-30T05:11:30.744506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Scheduler\nWe will train our model with a Step Train Schedule for 4 epochs. First 2 epochs are LR=1e-3. Then epochs 3 and 4 use LR=1e-4 and 1e-5 respectively. (Below we also provide a Cosine Train Schedule if you want to experiment with it. Note it is not used in this notebook).","metadata":{"papermill":{"duration":0.026741,"end_time":"2024-01-14T22:52:52.251841","exception":false,"start_time":"2024-01-14T22:52:52.2251","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import math\nLR_START = 1e-6\nLR_MAX = 1e-3\nLR_MIN = 1e-6\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS2 = 4\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS2)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y, '-o')\nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Cosine Training Schedule',size=16); plt.show()\n\nLR2 = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.304824,"end_time":"2024-01-14T22:52:52.58355","exception":false,"start_time":"2024-01-14T22:52:52.278726","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:11:30.746524Z","iopub.execute_input":"2024-04-30T05:11:30.746822Z","iopub.status.idle":"2024-04-30T05:11:31.051837Z","shell.execute_reply.started":"2024-04-30T05:11:30.746797Z","shell.execute_reply":"2024-04-30T05:11:31.050855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 1e-4\nLR_MAX = 1e-3\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.1\nEVERY = 1\nEPOCHS = 4\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//EVERY)\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, y, 'o-'); \nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Step Training Schedule',size=16); plt.show()\n\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.309296,"end_time":"2024-01-14T22:52:52.92271","exception":false,"start_time":"2024-01-14T22:52:52.613414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:11:31.053145Z","iopub.execute_input":"2024-04-30T05:11:31.053501Z","iopub.status.idle":"2024-04-30T05:11:31.348453Z","shell.execute_reply.started":"2024-04-30T05:11:31.053472Z","shell.execute_reply":"2024-04-30T05:11:31.347436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build EfficientNet Model\nVersion 1-3 uses EfficientNet B2. Version 4 uses EfficientNet B0. Our models receives both Kaggle spectrograms and EEG spectrograms from our data loader. We then reshape these 8 spectrograms into 1 large flat image and feed it into EfficientNet.","metadata":{"papermill":{"duration":0.027228,"end_time":"2024-01-14T22:52:52.97653","exception":false,"start_time":"2024-01-14T22:52:52.949302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install Keras-Applications\n!pip install efficientnet","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:31.352764Z","iopub.execute_input":"2024-04-30T05:11:31.353084Z","iopub.status.idle":"2024-04-30T05:11:56.469963Z","shell.execute_reply.started":"2024-04-30T05:11:31.353055Z","shell.execute_reply":"2024-04-30T05:11:56.468856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Model\nWe train using Group KFold on patient id. If `LOAD_MODELS_FROM = None`, then we will train new models in this notebook version. Otherwise we will load saved models from the path `LOAD_MODELS_FROM`.","metadata":{"papermill":{"duration":0.033717,"end_time":"2024-01-14T22:53:06.742557","exception":false,"start_time":"2024-01-14T22:53:06.70884","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.47151Z","iopub.execute_input":"2024-04-30T05:11:56.471814Z","iopub.status.idle":"2024-04-30T05:11:56.502826Z","shell.execute_reply.started":"2024-04-30T05:11:56.471787Z","shell.execute_reply":"2024-04-30T05:11:56.501796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\n\n# Split the data\nfrom sklearn.model_selection import train_test_split\n\ntrain, test = train_test_split(train, test_size=0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.504372Z","iopub.execute_input":"2024-04-30T05:11:56.505047Z","iopub.status.idle":"2024-04-30T05:11:56.519165Z","shell.execute_reply.started":"2024-04-30T05:11:56.505016Z","shell.execute_reply":"2024-04-30T05:11:56.518172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.520628Z","iopub.execute_input":"2024-04-30T05:11:56.521Z","iopub.status.idle":"2024-04-30T05:11:56.549777Z","shell.execute_reply.started":"2024-04-30T05:11:56.520969Z","shell.execute_reply":"2024-04-30T05:11:56.548764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.551025Z","iopub.execute_input":"2024-04-30T05:11:56.551361Z","iopub.status.idle":"2024-04-30T05:11:56.575006Z","shell.execute_reply.started":"2024-04-30T05:11:56.551333Z","shell.execute_reply":"2024-04-30T05:11:56.574138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\nfrom tensorflow.keras.applications import Xception, InceptionV3\n\ndef build_model(model_type):\n    inp = tf.keras.Input(shape=(128, 256, 8))\n\n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:, :, :, i:i + 1] for i in range(4)]\n    x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n\n    # EEG SPECTROGRAMS\n    x2 = [inp[:, :, :, i + 4:i + 5] for i in range(4)]\n    x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = tf.keras.layers.Concatenate(axis=2)([x1, x2])\n    elif USE_EEG_SPECTROGRAMS:\n        x = x2\n    else:\n        x = x1\n    x = tf.keras.layers.Concatenate(axis=3)([x, x, x])\n\n    if model_type == 'EfficientNet':\n        base_model = efn.EfficientNetB5(include_top=False, weights=\"imagenet\", input_shape=None)\n        base_model.trainable = False # Freeze pre-trained layers for fine-tuning\n    elif model_type == 'Xception':\n        base_model = Xception(include_top=False, weights=\"imagenet\", input_shape=None)\n        base_model.trainable = False  # Freeze pre-trained layers for fine-tuning\n    elif model_type == 'InceptionV3':\n        base_model = InceptionV3(include_top=False, weights=\"imagenet\", input_shape=None)\n        base_model.trainable = False  # Freeze pre-trained layers for fine-tuning\n\n    # OUTPUT\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6, activation='softmax', dtype='float32')(x)\n\n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer=opt)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.576267Z","iopub.execute_input":"2024-04-30T05:11:56.576636Z","iopub.status.idle":"2024-04-30T05:11:56.604961Z","shell.execute_reply.started":"2024-04-30T05:11:56.576603Z","shell.execute_reply":"2024-04-30T05:11:56.603968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\n# EfficientNet\nall_oof_effnet = []\nall_true_effnet = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=True)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model('EfficientNet')\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'EffNet_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n        \n    oof = model.predict(valid_gen, verbose=1)\n    all_oof_effnet.append(oof)\n    all_true_effnet.append(train.iloc[valid_index][TARGETS].values)\n    \n    del model, oof\n    gc.collect()\n    \nall_oof_effnet = np.concatenate(all_oof_effnet)\nall_true_effnet = np.concatenate(all_true_effnet)\n\n# Xception\nall_oof_xception = []\nall_true_xception = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=True)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model('Xception')\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'Xception_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}Xception_v{VER}_f{i}.h5')\n        \n    oof = model.predict(valid_gen, verbose=1)\n    all_oof_xception.append(oof)\n    all_true_xception.append(train.iloc[valid_index][TARGETS].values)\n    \n    del model, oof\n    gc.collect()\n    \nall_oof_xception = np.concatenate(all_oof_xception)\nall_true_xception = np.concatenate(all_true_xception)\n\n# InceptionV3\nall_oof_inceptionv3 = []\nall_true_inceptionv3 = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=True)\n    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model('InceptionV3')\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LR])\n        model.save_weights(f'InceptionV3_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}InceptionV3_v{VER}_f{i}.h5')\n        \n    oof = model.predict(valid_gen, verbose=1)\n    all_oof_inceptionv3.append(oof)\n    all_true_inceptionv3.append(train.iloc[valid_index][TARGETS].values)\n    \n    del model, oof\n    gc.collect()\n    \nall_oof_inceptionv3 = np.concatenate(all_oof_inceptionv3)\nall_true_inceptionv3 = np.concatenate(all_true_inceptionv3)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:11:56.606366Z","iopub.execute_input":"2024-04-30T05:11:56.606619Z","iopub.status.idle":"2024-04-30T05:51:21.646296Z","shell.execute_reply.started":"2024-04-30T05:11:56.606597Z","shell.execute_reply":"2024-04-30T05:51:21.645416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV Score for EfficientNet\nThis is CV score for our EfficientNet model.","metadata":{"papermill":{"duration":0.047893,"end_time":"2024-01-14T22:55:48.045405","exception":false,"start_time":"2024-01-14T22:55:47.997512","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\n# EfficientNet\noof = pd.DataFrame(all_oof_effnet)\noof['id'] = np.arange(len(oof))\ntrue = pd.DataFrame(all_true_effnet)\ntrue['id'] = np.arange(len(true))\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint(f'EfficientNet CV = {cv}')\n\n# Xception\noof = pd.DataFrame(all_oof_xception)\noof['id'] = np.arange(len(oof))\ntrue = pd.DataFrame(all_true_xception)\ntrue['id'] = np.arange(len(true))\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint(f'Xception CV = {cv}')\n\n# InceptionV3\noof = pd.DataFrame(all_oof_inceptionv3)\noof['id'] = np.arange(len(oof))\ntrue = pd.DataFrame(all_true_inceptionv3)\ntrue['id'] = np.arange(len(true))\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint(f'InceptionV3 CV = {cv}')","metadata":{"papermill":{"duration":0.126007,"end_time":"2024-01-14T22:55:48.222599","exception":false,"start_time":"2024-01-14T22:55:48.096592","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-04-30T05:51:21.64767Z","iopub.execute_input":"2024-04-30T05:51:21.64802Z","iopub.status.idle":"2024-04-30T05:51:21.731161Z","shell.execute_reply.started":"2024-04-30T05:51:21.647992Z","shell.execute_reply":"2024-04-30T05:51:21.730259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_gen = DataGenerator(test, shuffle=False, batch_size=100, mode='test')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:51:21.767873Z","iopub.execute_input":"2024-04-30T05:51:21.768212Z","iopub.status.idle":"2024-04-30T05:51:21.783674Z","shell.execute_reply.started":"2024-04-30T05:51:21.768185Z","shell.execute_reply":"2024-04-30T05:51:21.782833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:51:21.785087Z","iopub.execute_input":"2024-04-30T05:51:21.785584Z","iopub.status.idle":"2024-04-30T05:51:21.811131Z","shell.execute_reply.started":"2024-04-30T05:51:21.785548Z","shell.execute_reply":"2024-04-30T05:51:21.810258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['target'].values","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:51:21.812486Z","iopub.execute_input":"2024-04-30T05:51:21.812831Z","iopub.status.idle":"2024-04-30T05:51:21.822181Z","shell.execute_reply.started":"2024-04-30T05:51:21.812788Z","shell.execute_reply":"2024-04-30T05:51:21.821259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_gen = DataGenerator(test, shuffle=False, batch_size=1, mode='test')\ntrain_gen = DataGenerator(train, shuffle=False, batch_size=1, mode='test')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:51:21.823141Z","iopub.execute_input":"2024-04-30T05:51:21.823454Z","iopub.status.idle":"2024-04-30T05:51:21.83207Z","shell.execute_reply.started":"2024-04-30T05:51:21.823423Z","shell.execute_reply":"2024-04-30T05:51:21.831328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix, roc_curve, roc_auc_score, auc, accuracy_score, precision_score, recall_score, f1_score","metadata":{"execution":{"iopub.status.busy":"2024-04-30T05:52:14.506764Z","iopub.execute_input":"2024-04-30T05:52:14.507145Z","iopub.status.idle":"2024-04-30T05:52:14.51215Z","shell.execute_reply.started":"2024-04-30T05:52:14.507113Z","shell.execute_reply":"2024-04-30T05:52:14.511169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_model_name(model, train_gen, test_gen, name):\n    print('Evaluating Model:', name)\n    \n    Train Accuracy\n    y_true = train['target'].values\n    y_true = np.array([TARS[x] for x in y_true])\n    y_pred = model.predict(train_gen, verbose=1)\n    y_pred = y_pred / y_pred.sum(axis=1,keepdims=True)\n    y_pred = y_pred.argmax(axis=1)\n    print('Train Accuracy:', accuracy_score(y_true, y_pred))\n    \n    # Test Accuracy\n    y_true = test['target'].values\n    y_true = np.array([TARS[x] for x in y_true])\n    y_pred_prob = model.predict(test_gen, verbose=1)\n    y_pred_prob = y_pred_prob / y_pred_prob.sum(axis=1, keepdims=True)\n    y_pred = y_pred_prob.argmax(axis=1)\n    print('Test Accuracy:', accuracy_score(y_true, y_pred))\n    \n    # Precision, Recall, F1 Score\n    print('Precision : ', precision_score(y_true, y_pred, average='weighted', zero_division=1))\n    print('Recall : ', recall_score(y_true, y_pred, average='weighted', zero_division=1))\n    print('F1 Score : ', f1_score(y_true, y_pred, average='weighted', zero_division=1))\n    \n    # Confusion Matrix\n    print(classification_report(y_true, y_pred, target_names=[x for x in TARS.keys()]))\n\n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(8,6))\n    plt.imshow(cm, cmap=plt.cm.Blues)\n    for i in range(6):\n        for j in range(6):\n            plt.text(j, i, cm[i, j], ha='center', va='center', color='black')\n    plt.xlabel('Predicted labels',size=14)\n    plt.ylabel('True labels',size=14)\n    plt.xticks([0,1,2,3,4,5], [x for x in TARS.keys()], rotation=45)\n    plt.yticks([0,1,2,3,4,5], [x for x in TARS.keys()])\n    plt.colorbar()\n    plt.title('Confusion Matrix',size=16)\n    plt.show()    \n    \n    return y_true, y_pred, y_pred_prob\n    ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T06:02:00.380618Z","iopub.execute_input":"2024-04-30T06:02:00.381004Z","iopub.status.idle":"2024-04-30T06:02:00.394292Z","shell.execute_reply.started":"2024-04-30T06:02:00.380971Z","shell.execute_reply":"2024-04-30T06:02:00.393274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\n\n# ROC Curve and AUC Score for EfficientNet\nmodel = build_model('EfficientNet')\nmodel.load_weights(f'EffNet_v1_f0.h5')\ny_true_e, y_pred_e, y_pred_prob_e = evaluate_model_name(model, train_gen, test_gen, 'EfficientNetB5')\n\nfpr = dict()\ntpr = dict()\nroc_auc = dict()\nfor i in range(6):\n    fpr[i], tpr[i], _ = roc_curve(y_true_e == i, y_pred_prob_e[:, i])\n    roc_auc[i] = auc(fpr[i], tpr[i])\n    \nplt.figure(figsize=(8,6))\nfor i in range(6):\n    plt.plot(fpr[i], tpr[i], label=f'{TARS2[i]} (AUC = {roc_auc[i]:0.2f})')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlabel('False Positive Rate',size=14)\nplt.ylabel('True Positive Rate',size=14)\nplt.title('ROC Curve',size=16)\n\nplt.legend(loc=\"lower right\")\nplt.show()\n\n\n# ROC Curve and AUC Score for Xception  \nmodel = build_model('Xception')\nmodel.load_weights(f'Xception_v1_f0.h5')\ny_true_x, y_pred_x, y_pred_prob_x = evaluate_model_name(model, train_gen, test_gen, 'Xception')\n\nfpr = dict()\ntpr = dict()\nroc_auc = dict()\nfor i in range(6):\n    fpr[i], tpr[i], _ = roc_curve(y_true_x == i, y_pred_prob_x[:, i])\n    roc_auc[i] = auc(fpr[i], tpr[i])\n    \nplt.figure(figsize=(8,6))\nfor i in range(6):\n    plt.plot(fpr[i], tpr[i], label=f'{TARS2[i]} (AUC = {roc_auc[i]:0.2f})')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlabel('False Positive Rate',size=14)\nplt.ylabel('True Positive Rate',size=14)\nplt.title('ROC Curve',size=16)\n\nplt.legend(loc=\"lower right\")\nplt.show()\n\n# ROC Curve and AUC Score for InceptionV3\nmodel = build_model('InceptionV3')\nmodel.load_weights(f'InceptionV3_v1_f0.h5')\ny_true_i, y_pred_i, y_pred_prob_i = evaluate_model_name(model, train_gen, test_gen, 'InceptionV3')\n\nfpr = dict()\ntpr = dict()\nroc_auc = dict()\nfor i in range(6):\n    fpr[i], tpr[i], _ = roc_curve(y_true_i == i, y_pred_prob_i[:, i])\n    roc_auc[i] = auc(fpr[i], tpr[i])\n    \nplt.figure(figsize=(8,6))\nfor i in range(6):\n    plt.plot(fpr[i], tpr[i], label=f'{TARS2[i]} (AUC = {roc_auc[i]:0.2f})')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlabel('False Positive Rate',size=14)\nplt.ylabel('True Positive Rate',size=14)\nplt.title('ROC Curve',size=16)\n\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T06:55:44.774281Z","iopub.execute_input":"2024-04-30T06:55:44.775252Z","iopub.status.idle":"2024-04-30T06:55:52.990318Z","shell.execute_reply.started":"2024-04-30T06:55:44.775196Z","shell.execute_reply":"2024-04-30T06:55:52.989027Z"},"trusted":true},"execution_count":null,"outputs":[]}]}