{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7539864,"sourceType":"datasetVersion","datasetId":4390432},{"sourceId":7761782,"sourceType":"datasetVersion","datasetId":4431587},{"sourceId":10214913,"sourceType":"datasetVersion","datasetId":5433190},{"sourceId":158958765,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np, os, sys, json\nimport matplotlib.pyplot as plt\nimport mne\nimport seaborn as sns\nimport time\nimport tensorflow as tf\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedGroupKFold\nimport tensorflow.keras.backend as K, gc\nimport librosa\nimport math\nimport albumentations as albu\n\n# starter similar to https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43#Train-DataLoader\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n    \n# importing custom functions\n#! pip install --no-index --find-links /kaggle/input/preprocessingwheel4hms pyprep\n#from pyprep.find_noisy_channels import NoisyChannels\n\n# eff net working well apparently\n!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl\nimport efficientnet.tfkeras as efn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:11.879136Z","iopub.execute_input":"2025-02-12T14:51:11.879359Z","iopub.status.idle":"2025-02-12T14:51:21.842140Z","shell.execute_reply.started":"2025-02-12T14:51:11.879340Z","shell.execute_reply":"2025-02-12T14:51:21.841271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"tensorflow version: {tf.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:21.843258Z","iopub.execute_input":"2025-02-12T14:51:21.844179Z","iopub.status.idle":"2025-02-12T14:51:21.849113Z","shell.execute_reply.started":"2025-02-12T14:51:21.844156Z","shell.execute_reply":"2025-02-12T14:51:21.848270Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Starting with the actual code\n* Loading train set\n* DataGen\n* Networks and params\n* FUN!","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\n\ntrain = df.groupby('eeg_id')['spectrogram_id'].agg(idxSpec='first')\ntrain['patient_id'] = df.groupby('eeg_id')[['patient_id']].agg('first')\n\ndftemp=df.groupby('eeg_id')[TARGETS].agg('sum')\n\ntrain[TARGETS] = dftemp.apply(lambda x: (x / x.sum()), axis=1)\ntrain['numVotes'] = dftemp.apply(lambda x:  x.sum(), axis=1)\ntrain['target'] = df.groupby('eeg_id')[['expert_consensus']].agg('first')\n\ndef compute_entropy(probs):\n    return -np.sum(probs * np.log(probs + 1e-10), axis=1)  # Adding a small constant to avoid log(0)\n\ndef normalize_weights(weights):\n    return (weights - weights.min()) / (weights.max() - weights.min())\n\ndef invert_weights(weights):\n    return 1 - weights\n\ntrain['entropy'] = compute_entropy(train[TARGETS].values)\ntrain['normalized_entropy'] = normalize_weights(train['entropy'].values)\n# Invert the normalized entropy values\ntrain['inverted_entropy'] = invert_weights(train['normalized_entropy'])\n\ntrainHQ = train[train['numVotes']>20]\n\ntrain = train.reset_index()\ntrainHQ = trainHQ.reset_index()\nIDSubj = trainHQ.eeg_id.unique()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:21.851473Z","iopub.execute_input":"2025-02-12T14:51:21.851860Z","iopub.status.idle":"2025-02-12T14:51:24.665281Z","shell.execute_reply.started":"2025-02-12T14:51:21.851833Z","shell.execute_reply":"2025-02-12T14:51:24.664532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(train))\ntrain.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:24.666225Z","iopub.execute_input":"2025-02-12T14:51:24.666468Z","iopub.status.idle":"2025-02-12T14:51:24.685704Z","shell.execute_reply.started":"2025-02-12T14:51:24.666449Z","shell.execute_reply":"2025-02-12T14:51:24.684828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Selecting the patients","metadata":{}},{"cell_type":"code","source":"# # Step 1: Group the data by 'patient_id'\n# grouped = trainHQ.groupby('patient_id')\n\n# # Step 2: Calculate the number of instances per class for each patient\n# patient_class_counts = grouped['target'].value_counts().unstack(fill_value=0)\n# sortValCont = patient_class_counts.sort_values(by=['GPD'],ascending=False)\n\n# # Step 3: select grouping as needed\n# selected_patients=[]\n# lowValCont = patient_class_counts.query('GRDA <= 3 & GRDA >= 1')\n# RDAsel = lowValCont.query('LRDA <= 3 & LRDA >= 1')\n# selected_patients = selected_patients + RDAsel.index.values.tolist()\n\n# lowValCont = patient_class_counts.query('GPD <= 3 & GPD >= 1')\n# GPDsel = lowValCont.iloc[:50,:]\n# selected_patients = selected_patients + RDAsel.index.values.tolist()\n\n# lowValCont = patient_class_counts.query('LPD <= 3 & LPD >= 1')\n# PDsel = lowValCont.query('Seizure <= 3 & Seizure >= 1')\n# selected_patients = selected_patients + PDsel.index.values.tolist()\n\n# trainSel = trainHQ[~trainHQ['patient_id'].isin(selected_patients)]\n# test_gen = trainHQ[trainHQ['patient_id'].isin(selected_patients)]\n\n# print(\"###############################\")\n# class_counts = trainSel['target'].value_counts()\n# print(\"Train Set Class Distribution:\")\n# print(class_counts)\n# print(\"Training size patients :\")\n# print(len(trainSel.patient_id.unique()))\n# print(\"Training size total :\")\n# print(len(trainSel))\n\n# print(\"###############################\")\n# class_counts = test_gen['target'].value_counts()\n# print(\"Validation Set Class Distribution:\")\n# print(class_counts)\n# print(\"Number of patients:\")\n# print(len(test_gen.patient_id.unique()))\n\n\n# # Check for common values in 'patient_id' column\n# common_values = trainSel['patient_id'].isin(test_gen['patient_id'])\n\n# # Get common patient_ids\n# common_patient_ids = trainSel.loc[common_values, 'patient_id'].unique()\n\n# print(\"Common patient IDs:\", len(common_patient_ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:24.686733Z","iopub.execute_input":"2025-02-12T14:51:24.686971Z","iopub.status.idle":"2025-02-12T14:51:24.691606Z","shell.execute_reply.started":"2025-02-12T14:51:24.686952Z","shell.execute_reply":"2025-02-12T14:51:24.690739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gkf = StratifiedGroupKFold(n_splits=8)\ncntstrat=1\n\nfor i, (train_index, valid_index) in enumerate(gkf.split(trainHQ, trainHQ.target, trainHQ.patient_id)): \n    \n    cntstrat=cntstrat+1\n\n\ntrainSel = trainHQ.iloc[train_index]\ntest_gen = trainHQ.iloc[valid_index]\n\nprint(\"###############################\")\nclass_counts = trainSel['target'].value_counts()\nprint(\"Train Set Class Distribution:\")\nprint(class_counts)\nprint(\"Training size patients :\")\nprint(len(trainSel.patient_id.unique()))\nprint(\"Training size total :\")\nprint(len(trainSel))\n\nprint(\"###############################\")\nclass_counts = test_gen['target'].value_counts()\nprint(\"Validation Set Class Distribution:\")\nprint(class_counts)\nprint(\"Number of patients:\")\nprint(len(test_gen.patient_id.unique()))\n\n# Check for common values in 'patient_id' column\ncommon_values = trainSel['patient_id'].isin(test_gen['patient_id'])\n\n# Get common patient_ids\ncommon_patient_ids = trainSel.loc[common_values, 'patient_id'].unique()\n\nprint(\"Common patient IDs:\", len(common_patient_ids))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:24.692530Z","iopub.execute_input":"2025-02-12T14:51:24.692778Z","iopub.status.idle":"2025-02-12T14:51:25.573621Z","shell.execute_reply.started":"2025-02-12T14:51:24.692746Z","shell.execute_reply":"2025-02-12T14:51:25.572791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#trainSel = trainHQ[:trainNum]\n#test_gen = trainHQ[trainNum:]\ntest_gen = test_gen.reset_index()\ny_true=test_gen[TARGETS].to_numpy()\n#test_gen[TARGETS].head(2)\ntest_gen.to_csv('testSet.csv', index=False)\n\ndel trainHQ\ndel dftemp\ndel df\ndel train\n#del sortValCont, patient_class_counts, grouped\n#del selected_patients, GPDsel, RDAsel, PDsel\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:25.575090Z","iopub.execute_input":"2025-02-12T14:51:25.575464Z","iopub.status.idle":"2025-02-12T14:51:25.805207Z","shell.execute_reply.started":"2025-02-12T14:51:25.575435Z","shell.execute_reply":"2025-02-12T14:51:25.804344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    # adapted from https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43#Kaggle-Spectrograms-vs.-EEG-Spectrograms\n    # and also https://www.kaggle.com/code/awsaf49/hms-hbac-kerascv-starter-notebook\n    # removing agumentation... maybe done in future but still on separate dataset creation\n    def __init__(self, data, batch_size=32, shuffle = False, augment=False, DataSetType='training'): \n        'Generates data for Keras'\n        self.data = data\n        self.batch_size = batch_size\n        self.type = DataSetType\n        self.shuffle = shuffle\n        self.augment = augment\n        self.on_epoch_end()\n        \n        \n    def __len__(self):\n        'Denotes the number of batches'\n        return int(np.ceil(self.data.shape[0]/self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y, w = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X) \n        return X, y, w\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,12),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        w = np.zeros((len(indexes),),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            \n            for k in range(4):\n                # ORIGINAL SPECTROGRAMS\n                TMPSPEC = np.load(f'/kaggle/input/newdatahmschallenge/{row.eeg_id}_transEEG_Data.npy',allow_pickle=True).item().get('spec')\n                X[j,:,:,k] = TMPSPEC[k,:,:]\n\n                # EEG SPECTROGRAMS\n                TMPEEG = np.load(f'/kaggle/input/newdatahmschallenge/{row.eeg_id}_transEEG_Data.npy',allow_pickle=True).item().get('specEEG')\n                X[j,:,:,k+4] = TMPEEG[k,:,:]\n                TMPEEGtr = np.load(f'/kaggle/input/newdatahmschallenge/{row.eeg_id}_transEEG_Data.npy',allow_pickle=True).item().get('specEEGtrans')\n                X[j,:,:,k+8] = TMPEEGtr[k,:,:]*4/3\n                \n            if self.type!='test':\n                y[j,] = row[TARGETS]\n                w[j,] = row['inverted_entropy']\n                \n        return X,y,w\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5), \n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:25.806272Z","iopub.execute_input":"2025-02-12T14:51:25.806530Z","iopub.status.idle":"2025-02-12T14:51:25.817599Z","shell.execute_reply.started":"2025-02-12T14:51:25.806509Z","shell.execute_reply":"2025-02-12T14:51:25.816722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gen = DataGenerator(trainSel, batch_size=32)\nROWS=2; COLS=3; BATCHES=2\n\ncheckPlotGen=1\n\nif checkPlotGen:\n    for i,(x,y,w) in enumerate(gen):\n        plt.figure(figsize=(20,8))\n        for j in range(ROWS):\n            for k in range(COLS):\n                plt.subplot(ROWS,COLS,j*COLS+k+1)\n                t = y[j*COLS+k]\n                img = x[j*COLS+k,:,:,4][::-1,]\n                mn = img.flatten().min()\n                mx = img.flatten().max()\n                img = (img-mn)/(mx-mn)\n                plt.imshow(img)\n                tars = f'[{t[0]:0.2f}'\n                for s in t[1:]: tars += f', {s:0.2f}'\n                eeg = trainSel.eeg_id.values[i*32+j*COLS+k]\n                plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n                plt.yticks([])\n                plt.ylabel('Frequencies (Hz)',size=14)\n                plt.xlabel('Time',size=16)\n        plt.show()\n        \n        #plt.figure()\n        #plt.boxplot(img.flatten())\n        #plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n        \n        if i==BATCHES-1: break\n\ndel gen\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:25.820831Z","iopub.execute_input":"2025-02-12T14:51:25.821166Z","iopub.status.idle":"2025-02-12T14:51:30.408463Z","shell.execute_reply.started":"2025-02-12T14:51:25.821145Z","shell.execute_reply":"2025-02-12T14:51:30.407532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport psutil\n\ndef cpu_stats():\n    pid = os.getpid()\n    py = psutil.Process(pid)\n    memory_use = py.memory_info()[0] / 2. ** 30\n    return 'memory GB:' + str(np.round(memory_use, 2))\n    \ncpu_stats()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:30.409526Z","iopub.execute_input":"2025-02-12T14:51:30.409796Z","iopub.status.idle":"2025-02-12T14:51:30.416845Z","shell.execute_reply.started":"2025-02-12T14:51:30.409774Z","shell.execute_reply":"2025-02-12T14:51:30.416033Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Setting up the network","metadata":{}},{"cell_type":"code","source":"EPOCHS=8\n\ninitial_learning_rate = 1e-3\nstartDecay = 2\ndef lr_const(epoch):\n    lr=initial_learning_rate\n    if epoch > startDecay:\n        if epoch > startDecay+1:\n            if epoch > startDecay+2:\n                if epoch > startDecay+3:\n                    if epoch > startDecay+4:\n                        lr=lr/100000\n                    else:\n                        lr=lr/10000\n                else:\n                    lr=lr/1000\n            else:\n                lr=lr/100\n        else:\n            lr=lr/10 \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lr_const(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, np.log(y), 'o-'); \nplt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\nplt.title('Step Training Schedule',size=16); plt.show()\n\nLRMAN = tf.keras.callbacks.LearningRateScheduler(lr_const, verbose = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:30.417837Z","iopub.execute_input":"2025-02-12T14:51:30.418096Z","iopub.status.idle":"2025-02-12T14:51:30.592285Z","shell.execute_reply.started":"2025-02-12T14:51:30.418076Z","shell.execute_reply":"2025-02-12T14:51:30.591385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_model():\n    \n    inp = tf.keras.Input(shape=(128,256,12))\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')   \n    \n    # bad coding but prefer to fix it here\n    typeFeatAn = 1\n    \n    if typeFeatAn == 0:\n        x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n        x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n        x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n        x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n        x3 = [inp[:,:,:,i+8:i+9] for i in range(4)]\n        x3 = tf.keras.layers.Concatenate(axis=1)(x3)\n        \n        x = tf.keras.layers.Concatenate(axis=2)([x1,x2,x3])\n        x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    if typeFeatAn == 1:\n        # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n        # SPECTROGRAMS\n        x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n        x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n        x3 = [inp[:,:,:,i+8:i+9] for i in range(4)]\n        x3 = tf.keras.layers.Concatenate(axis=1)(x3)\n\n        # MAKE 512X512X3\n        x = tf.keras.layers.Concatenate(axis=2)([x2,x3])\n        x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n\n    # OUTPUT\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt) \n        \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:30.593342Z","iopub.execute_input":"2025-02-12T14:51:30.593644Z","iopub.status.idle":"2025-02-12T14:51:30.603942Z","shell.execute_reply.started":"2025-02-12T14:51:30.593622Z","shell.execute_reply":"2025-02-12T14:51:30.602935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = build_model()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:30.605159Z","iopub.execute_input":"2025-02-12T14:51:30.605448Z","iopub.status.idle":"2025-02-12T14:51:33.495569Z","shell.execute_reply.started":"2025-02-12T14:51:30.605423Z","shell.execute_reply":"2025-02-12T14:51:33.494648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def expected_calibration_error(samples, true_labels_strings, M=10):\n\n    if isinstance(true_labels_strings[0], str):\n        mapping = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4,'Other':5}\n\n        true_labels = [mapping[label] for label in true_labels_strings]\n    else:\n        true_labels = true_labels_strings\n    \n    # uniform binning approach with M number of bins\n    bin_boundaries = np.linspace(0, 1, M + 1)\n    bin_lowers = bin_boundaries[:-1]\n    bin_uppers = bin_boundaries[1:]\n\n    # get max probability per sample i\n    confidences = np.max(samples, axis=1)\n    # get predictions from confidences (positional in this case)\n    predicted_label = np.argmax(samples, axis=1)\n\n    # get a boolean list of correct/false predictions\n    accuracies = predicted_label==true_labels\n\n    ece = np.zeros(1)\n    for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):\n        # determine if sample is in bin m (between bin lower &amp; upper)\n        in_bin = np.logical_and(confidences > bin_lower.item(), confidences <= bin_upper.item())\n        # can calculate the empirical probability of a sample falling into bin m: (|Bm|/n)\n        prob_in_bin = in_bin.mean()\n\n        if prob_in_bin.item() > 0:\n            # get the accuracy of bin m: acc(Bm)\n            accuracy_in_bin = accuracies[in_bin].mean()\n            # get the average confidence of bin m: conf(Bm)\n            avg_confidence_in_bin = confidences[in_bin].mean()\n            # calculate |acc(Bm) - conf(Bm)| * (|Bm|/n) for bin m and add to the total ECE\n            ece += np.abs(avg_confidence_in_bin - accuracy_in_bin) * prob_in_bin\n    return ece","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:51:33.496671Z","iopub.execute_input":"2025-02-12T14:51:33.496933Z","iopub.status.idle":"2025-02-12T14:51:33.503968Z","shell.execute_reply.started":"2025-02-12T14:51:33.496912Z","shell.execute_reply":"2025-02-12T14:51:33.503044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nres_unique_labels = ['Seizure', 'LPD', 'GPD', 'LRDA', 'GRDA','Other']\n\n# Function to compute Percent Agreement (PA) for Majority IRR\ndef percent_agreement(hard_labels, true_labels):\n    return np.mean(np.array(hard_labels) == np.array(true_labels))\n\n# Function to compute Kappa (κ) for Majority IRR\ndef kappa(hard_labels, true_labels):\n    PA = percent_agreement(hard_labels, true_labels)\n    PC = 1 / 6  # Chance agreement (since there are 6 classes)\n    return (PA - PC) / (1 - PC)\n\n# Function to apply the classification rule with given thresholds\ndef apply_classification(oof_simul, thresholds, res_unique_labels):\n    hard_labels = []\n    for row in oof_simul:\n        sorted_indices = np.argsort(row)[::-1]  # Sort descending\n        best_class = sorted_indices[0]\n        second_best_class = sorted_indices[1]\n        \n        best_prob = row[best_class]\n        second_best_prob = row[second_best_class]\n        \n        threshold_key = (best_class, second_best_class)\n        threshold = thresholds.get(threshold_key, 0)  # Default to 0 if no threshold defined\n        \n        if (best_prob - second_best_prob) < threshold:\n            hard_labels.append(res_unique_labels[second_best_class])\n        else:\n            hard_labels.append(res_unique_labels[best_class])\n    \n    return hard_labels\n\n# Function to perform random search and calculate Majority IRR\ndef random_search(oof_simul, true_labels, num_simulations=10000):\n    thresholds = {\n        (0, 1): 0,  # SZ vs LPD\n        (0, 2): 0,  # SZ vs GPD\n        (0, 3): 0, # SZ vs LRDA\n        \n        (1, 0): 0,  \n        (1, 2): 0,  \n        (1, 3): 0, \n        \n        (2, 0): 0,  \n        (2, 1): 0,  \n        (2, 4): 0, \n        \n        (3, 0): 0,  \n        (3, 1): 0,  \n        (3, 4): 0, \n\n        (4, 1): 0,  \n        (4, 2): 2,  \n        (4, 3): 3, \n        (4, 3): 5, \n        \n        (5, 0): 0,  \n        (5, 1): 0,  \n        (5, 2): 0, \n        (5, 3): 0,\n        (5, 4): 0,\n    }\n    \n    best_kappa = -1\n    best_thresholds = None\n\n    for cntsim in range(num_simulations):\n        # Generate random thresholds for each pair (if needed)\n        random_thresholds = {key: np.random.uniform(0, 1) for key in thresholds.keys()}\n        \n        # Apply classification with the current set of thresholds\n        hard_labels = apply_classification(oof_simul, random_thresholds, res_unique_labels)\n        \n        # Calculate Kappa for Majority IRR\n        current_kappa = kappa(hard_labels, true_labels)\n        \n        # Update best Kappa if this one is better\n        if current_kappa > best_kappa:\n            best_kappa = current_kappa\n            best_thresholds = random_thresholds\n\n        if cntsim % 10000 == 0:\n            print(f\"Done with {cntsim} sim\")\n    \n    return best_kappa, best_thresholds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T15:27:10.222790Z","iopub.execute_input":"2025-02-12T15:27:10.223423Z","iopub.status.idle":"2025-02-12T15:27:10.234966Z","shell.execute_reply.started":"2025-02-12T15:27:10.223395Z","shell.execute_reply":"2025-02-12T15:27:10.234124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_oof = []\nall_true = []\nSVC_foldsMod = []\nSVC_foldsMod_RDA = []\n\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import calibration_curve\n\n\nNFolds = 5\nVER = 4\nnum_classes = 6\nnbinsCalib = 5\n\n#LOAD_MODELS_FROM = None\nLOAD_MODELS_FROM = '/kaggle/input/weffnet-train-20240807/'\n\ngkf = StratifiedGroupKFold(n_splits=NFolds)\ntuning=1\nall_eval = np.empty((NFolds,))\n\nlabel_encoder = LabelEncoder()\n\nencoded_GRDA = 1 \nencoded_LRDA = 3  \nnew_class = 3\nfig, axes = plt.subplots(nrows=2, ncols=3, figsize=(18, 10))\naxes = axes.flatten()\n\n\n# setting np arrays for calibrations curves\ncalibration_list_x = np.empty((NFolds,num_classes,nbinsCalib))\ncalibration_list_y = np.empty((NFolds,num_classes,nbinsCalib))\n\nece_per_class = np.empty((NFolds,num_classes))\n\nflagTrainSVC = False\nfolds_bestThresh = []\nfor i, (train_index, valid_index) in enumerate(gkf.split(trainSel, trainSel.target, trainSel.patient_id)):  \n    \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    ax = axes[i]\n    \n    train_gen = DataGenerator(trainSel.iloc[train_index], shuffle=False, batch_size=16)\n    valid_gen = DataGenerator(trainSel.iloc[valid_index], shuffle=False, batch_size=32)\n    \n    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n    print('#'*25)\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n    if LOAD_MODELS_FROM is None:\n        model.fit(train_gen, verbose=1,\n              validation_data = valid_gen,\n              epochs=EPOCHS, callbacks = [LRMAN])\n        model.save_weights(f'EffNetTran_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNetTran_v{VER}_f{i}.h5')\n\n    # get scores\n    oof = model.predict(valid_gen, verbose=1)\n    if flagTrainSVC:\n        oofTrain = model.predict(train_gen)\n    \n    # assessing multiclass\n    true_labels_train = trainSel.iloc[train_index]['target'].values\n    true_labels_valid = trainSel.iloc[valid_index]['target'].values\n\n    #eval\n    scoreFold = model.evaluate(valid_gen)\n    print(f\"KL Divergence for this fold is: {scoreFold}\")\n    all_eval[i] = scoreFold\n\n    #########################\n    ### Calibration\n    ### Calibration curves\n    #########################\n\n    for jj in range(num_classes):\n        y_true = true_labels_valid == res_unique_labels[jj]\n        y_pred_class = oof[:, jj]\n    \n        prob_true, prob_pred = calibration_curve(y_true.astype(int), y_pred_class, n_bins=nbinsCalib, strategy='uniform')\n        calibration_list_x[i,jj,:] = prob_pred\n        calibration_list_y[i,jj,:] = prob_true\n        ax.plot(prob_pred, prob_true, marker='o', label=f\"{res_unique_labels[jj]}\")\n\n    #########################\n    ### Calibration\n    ### Expected Calibration Error per class\n    #########################\n    \n    bin_edges = np.linspace(0, 1, nbinsCalib + 1)\n    for class_idx in range(len(TARGETS)):\n        \n        accuracies = true_labels_valid == res_unique_labels[class_idx]\n        confidences = oof[:, class_idx]\n    \n        ece = 0\n        for kkk in range(nbinsCalib):\n            bin_lower, bin_upper = bin_edges[kkk], bin_edges[kkk + 1]\n            in_bin = (confidences > bin_lower) & (confidences <= bin_upper)\n            if np.sum(in_bin) > 0:\n                bin_acc = np.mean(accuracies[in_bin])\n                bin_conf = np.mean(confidences[in_bin])\n                ece += np.abs(bin_acc - bin_conf) * np.sum(in_bin) / len(y_true)\n    \n        ece_per_class[i,class_idx] = ece\n\n    #########################\n    ### Calibration\n    ### Expected Calibration Error\n    #########################\n\n    eceFold = expected_calibration_error(oof, true_labels_valid, M=10)\n    print(f\"ECE for this fold is: {eceFold}\")\n    \n    #########################\n    ### Calibration with second best\n    ### Perform actual calibration and get new Expected Calibration Error TBD\n    #########################\n\n    mapping = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4,'Other':5}\n\n    true_labels_remapped = [mapping[label] for label in true_labels_valid]\n    \n    best_kappa, best_thresholds = random_search(oof, true_labels_remapped, num_simulations=100000)\n\n    folds_bestThresh.append(best_thresholds)\n    print(f\"ECE for this fold is: {best_thresholds}\")\n\n    if flagTrainSVC:\n        #########################\n        ### Compute SVC per fold\n        #########################  \n        # svc multiclass\n        svc_model = LinearSVC(random_state=0,multi_class=\"ovr\")\n        svc_model.fit(oofTrain, labels_encoded_train)\n        svc_model.score(oof,labels_encoded_valid)\n    \n        SVC_foldsMod.append(svc_model)\n        \n        # test on leftout\n        dt_pred = svc_model.predict(oof)\n        fold_accuracy = accuracy_score(labels_encoded_valid, dt_pred)\n        #print(f'Fold {i+1} Accuracy: {fold_accuracy:.4f}')\n        \n        # Calculate TPR and FPR for each class\n        # confusion matrix\n        cm = confusion_matrix(labels_encoded_valid, dt_pred)\n        num_classes = cm.shape[0]\n        for class_idx in range(len(TARGETS)):\n            \n            TP = cm[class_idx, class_idx]  # True Positives \n            FN = cm[class_idx, :].sum() - TP  # False Negatives \n            FP = cm[:, class_idx].sum() - TP  # False Positives \n            TN = cm.sum() - (FP + FN + TP)  # True Negatives\n            \n            # TPR and FPR\n            TPR = TP / (TP + FN) if (TP + FN) != 0 else 0\n            FPR = FP / (FP + TN) if (FP + TN) != 0 else 0\n            \n            #print(f'Class {label_encoder.classes_[class_idx]} - True Positive Rate (TPR): {TPR:.4f}, False Positive Rate (FPR): {FPR:.4f}')\n\n        \n    all_oof.append(oof)\n    all_true.append(trainSel.iloc[valid_index][TARGETS].values)\n    \n\n    if tuning==0:\n        break\n    else:\n        if flagTrainSVC:\n            del model, oof, dt_pred,svc_model\n        else:\n            del model, oof\n        gc.collect()\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T15:27:11.838708Z","iopub.execute_input":"2025-02-12T15:27:11.839495Z","iopub.status.idle":"2025-02-12T15:38:22.684163Z","shell.execute_reply.started":"2025-02-12T15:27:11.839461Z","shell.execute_reply":"2025-02-12T15:38:22.683309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"keys = list(folds_bestThresh[0].keys())\n\ndf = pd.DataFrame({\n    'Pair': [str(key) for key in keys],\n    'Threshold 0': np.array(list(folds_bestThresh[0].values())),\n    'Threshold 1': np.array(list(folds_bestThresh[1].values())),\n    'Threshold 2': np.array(list(folds_bestThresh[2].values())),\n    'Threshold 3': np.array(list(folds_bestThresh[3].values())),\n    'Threshold 4': np.array(list(folds_bestThresh[4].values()))\n})\n\ndf.set_index('Pair', inplace=True)\n\n# Create a heatmap to visualize changes in threshold values\nplt.figure(figsize=(10, 6))\nsns.heatmap(df.T, annot=True, cmap='coolwarm', fmt='.2f', cbar_kws={'label': 'Threshold Value'})\nplt.title('Thresholds Comparison Across Cross-Validation Runs')\nplt.xlabel('Class Pair')\nplt.ylabel('Threshold Set')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T15:45:54.968480Z","iopub.execute_input":"2025-02-12T15:45:54.968810Z","iopub.status.idle":"2025-02-12T15:45:55.410960Z","shell.execute_reply.started":"2025-02-12T15:45:54.968786Z","shell.execute_reply":"2025-02-12T15:45:55.410184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\nthresholdsFold = []\nfor i in range(NFolds):\n    thresholdsFold.append(np.array(list(folds_bestThresh[i].values())))\n\n# Convert to numpy arrays for easier median calculation\nthresholds = np.array(thresholdsFold)\n\nprint(thresholds.shape)\n# Calculate the median for each class pair\ncombined_median_thresholds = {}\n\nfor i, key in enumerate(folds_bestThresh[0].keys()):\n    combined_median_thresholds[key] = np.median(thresholds[:, i])\n\nprint(combined_median_thresholds)\n\nwith open('dict_thresholdsFolds.pkl','wb') as f:\n    pickle.dump(folds_bestThresh,f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T15:59:51.001842Z","iopub.execute_input":"2025-02-12T15:59:51.002253Z","iopub.status.idle":"2025-02-12T15:59:51.012414Z","shell.execute_reply.started":"2025-02-12T15:59:51.002224Z","shell.execute_reply":"2025-02-12T15:59:51.011462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calibX = np.mean(calibration_list_x, axis=0)\ncalibY = np.mean(calibration_list_y, axis=0)\ncalibY.shape\n\nfor jj in range(num_classes):\n    plt.plot(calibX[jj,:], calibY[jj,:], marker='o', label=f\"{res_unique_labels[jj]}\")\n    plt.savefig(f\"/kaggle/working/Full_Calibration.svg\")\nplt.show()\n\n#np.save('/kaggle/working/Full_ECE.npy', ece_per_class)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:52:23.362982Z","iopub.status.idle":"2025-02-12T14:52:23.363305Z","shell.execute_reply.started":"2025-02-12T14:52:23.363170Z","shell.execute_reply":"2025-02-12T14:52:23.363183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score as scoreKL\n\ncvMean=[]\n\nfor i in range(5):\n\n    oof = pd.DataFrame(all_oof[i].copy())\n    oof['id'] = np.arange(len(oof))\n\n    true = pd.DataFrame(all_true[i].copy())\n    true['id'] = np.arange(len(true))\n    cv = scoreKL(solution=true, submission=oof, row_id_column_name='id')\n\n    cvMean.append(cv)\n    \n    \n#print('CV Score KL-Div for EfficientNetB0 avg =',np.mean(cvMean))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:52:23.365068Z","iopub.status.idle":"2025-02-12T14:52:23.365403Z","shell.execute_reply.started":"2025-02-12T14:52:23.365263Z","shell.execute_reply":"2025-02-12T14:52:23.365277Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save","metadata":{}},{"cell_type":"code","source":"# save\nwith open('model_SVC.pkl','wb') as f:\n    pickle.dump(SVC_foldsMod,f)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-12T14:52:23.366341Z","iopub.status.idle":"2025-02-12T14:52:23.366602Z","shell.execute_reply.started":"2025-02-12T14:52:23.366477Z","shell.execute_reply":"2025-02-12T14:52:23.366488Z"}},"outputs":[],"execution_count":null}]}