{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":2378330,"sourceType":"datasetVersion","datasetId":492658},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":7601534,"sourceType":"datasetVersion","datasetId":4425227},{"sourceId":7754261,"sourceType":"datasetVersion","datasetId":4532886},{"sourceId":8027267,"sourceType":"datasetVersion","datasetId":4673903},{"sourceId":8028982,"sourceType":"datasetVersion","datasetId":4533761},{"sourceId":158958765,"sourceType":"kernelVersion"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TRAINING","metadata":{}},{"cell_type":"markdown","source":"## Global constants","metadata":{}},{"cell_type":"code","source":"# Set to True for inference only, False for training\nONLY_INFERENCE = True\n\n# Configuration for model training\nFOLDS = 5\nEPOCHS = 4\nBATCH = 32\nNAME = 'None'\n\nSPEC_SIZE  = (512, 512, 3)\nCLASSES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\nN_CLASSES = len(CLASSES)\nTARGETS = CLASSES\nimport matplotlib.pyplot as plt\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:40:44.607834Z","iopub.execute_input":"2024-03-30T05:40:44.608301Z","iopub.status.idle":"2024-03-30T05:40:44.621459Z","shell.execute_reply.started":"2024-03-30T05:40:44.60826Z","shell.execute_reply":"2024-03-30T05:40:44.620479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"IMPORTS","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:40:44.623219Z","iopub.execute_input":"2024-03-30T05:40:44.623567Z","iopub.status.idle":"2024-03-30T05:40:58.65416Z","shell.execute_reply.started":"2024-03-30T05:40:44.623535Z","shell.execute_reply":"2024-03-30T05:40:58.653258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport random\nimport sys\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom tensorflow.keras import backend as K\nfrom tqdm import tqdm \nfrom scipy.ndimage import gaussian_filter\nfrom scipy.signal import butter, filtfilt, iirnotch\nfrom scipy.signal import spectrogram as spectrogram_np\n\nimport efficientnet.tfkeras as efn\n\nsys.path.append(f'/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:40:58.655727Z","iopub.execute_input":"2024-03-30T05:40:58.656107Z","iopub.status.idle":"2024-03-30T05:41:19.597858Z","shell.execute_reply.started":"2024-03-30T05:40:58.656072Z","shell.execute_reply":"2024-03-30T05:41:19.59687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!nvidia-smi\n\n# Installation of RAPIDS to Use cuSignal\n!cp ../input/rapids/rapids.0.17.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\n!rm /opt/conda/envs/rapids.tar.gz\n\nsys.path += [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"]\nsys.path += [\"/opt/conda/envs/rapids/lib/python3.7\"]\nsys.path += [\"/opt/conda/envs/rapids/lib\"]\n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/\n\nimport cupy as cp\nimport cusignal","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:41:19.600306Z","iopub.execute_input":"2024-03-30T05:41:19.601017Z","iopub.status.idle":"2024-03-30T05:42:49.500129Z","shell.execute_reply.started":"2024-03-30T05:41:19.600981Z","shell.execute_reply":"2024-03-30T05:42:49.499088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Environment functions","metadata":{}},{"cell_type":"code","source":"# Set the visible CUDA devices\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n\n# Set the strategy for using GPUs\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus) <= 1:\n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse:\n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n\n# Configure memory growth\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n    except RuntimeError as e:\n        print(e)\n\n# Enable or disable mixed precision\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:49.501385Z","iopub.execute_input":"2024-03-30T05:42:49.501695Z","iopub.status.idle":"2024-03-30T05:42:50.708556Z","shell.execute_reply.started":"2024-03-30T05:42:49.501664Z","shell.execute_reply":"2024-03-30T05:42:50.707535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to set random seed for reproducibility\ndef set_random_seed(seed: int = 42, deterministic: bool = False):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    tf.random.set_seed(seed)\n    if deterministic:\n        os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    else:\n        os.environ.pop('TF_DETERMINISTIC_OPS', None)\n\n# Set a deterministic behavior\nset_random_seed(deterministic=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.709847Z","iopub.execute_input":"2024-03-30T05:42:50.710136Z","iopub.status.idle":"2024-03-30T05:42:50.885302Z","shell.execute_reply.started":"2024-03-30T05:42:50.710112Z","shell.execute_reply":"2024-03-30T05:42:50.884558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"def create_train_data():\n    # Read the dataset\n    df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n    \n    # Create a new identifier combining multiple columns\n    id_cols = ['eeg_id', 'spectrogram_id', 'seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n    df['new_id'] = df[id_cols].astype(str).agg('_'.join, axis=1)\n    \n    # Calculate the sum of votes for each class\n    df['sum_votes'] = df[CLASSES].sum(axis=1)\n    \n    # Group the data by the new identifier and aggregate various features\n    agg_functions = {\n        'eeg_id': 'first',\n        'eeg_label_offset_seconds': ['min', 'max'],\n        'spectrogram_label_offset_seconds': ['min', 'max'],\n        'spectrogram_id': 'first',\n        'patient_id': 'first',\n        'expert_consensus': 'first',\n        **{col: 'sum' for col in CLASSES},\n        'sum_votes': 'sum',\n    }\n    grouped_df = df.groupby('new_id').agg(agg_functions).reset_index()\n\n    # Flatten the MultiIndex columns and adjust column names\n    grouped_df.columns = [f\"{col[0]}_{col[1]}\" if col[1] else col[0] for col in grouped_df.columns]\n    grouped_df.columns = grouped_df.columns.str.replace('_first', '').str.replace('_sum', '')\n    \n    # Normalize the class columns\n    y_data = grouped_df[CLASSES].values\n    y_data_normalized = y_data / y_data.sum(axis=1, keepdims=True)\n    grouped_df[CLASSES] = y_data_normalized\n\n    # Split the dataset into high and low quality based on the sum of votes\n    high_quality_df = grouped_df[grouped_df['sum_votes'] >= 10].reset_index(drop=True)\n    low_quality_df = grouped_df[(grouped_df['sum_votes'] < 10) & (grouped_df['sum_votes'] >= 0)].reset_index(drop=True)\n\n    return high_quality_df, low_quality_df","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.886439Z","iopub.execute_input":"2024-03-30T05:42:50.886788Z","iopub.status.idle":"2024-03-30T05:42:50.89708Z","shell.execute_reply.started":"2024-03-30T05:42:50.886763Z","shell.execute_reply":"2024-03-30T05:42:50.896241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n\n    def __init__(self, data, batch_size=32, shuffle=False, mode='train'):\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.mode = mode\n        self.on_epoch_end()\n\n    def __len__(self):\n        \"\"\"Denotes the number of batches per epoch.\"\"\"\n        return int(np.ceil(len(self.data) / self.batch_size))\n\n    def __getitem__(self, index):\n        \"\"\"Generate one batch of data.\"\"\"\n        indexes = self.indexes[index * self.batch_size : (index + 1) * self.batch_size]\n        X, y = self.__data_generation(indexes)\n        return X, y\n\n    def on_epoch_end(self):\n        \"\"\"Updates indexes after each epoch.\"\"\"\n        self.indexes = np.arange(len(self.data))\n        if self.shuffle:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, indexes):\n        \"\"\"Generates data containing batch_size samples.\"\"\"\n        # Initialization\n        X = np.zeros((len(indexes), *SPEC_SIZE), dtype='float32')\n        y = np.zeros((len(indexes), len(CLASSES)), dtype='float32')\n\n        # Generate data\n        for j, i in enumerate(indexes):\n            row = self.data.iloc[i]\n            eeg_id = row['eeg_id']\n            spec_offset = int(row['spectrogram_label_offset_seconds_min'])\n            eeg_offset = int(row['eeg_label_offset_seconds_min'])\n            file_path = f'/kaggle/input/3-diff-time-specs-hms/images/{eeg_id}_{spec_offset}_{eeg_offset}.npz'\n            data = np.load(file_path)\n            eeg_data = data['final_image']\n            eeg_data_expanded = np.repeat(eeg_data[:, :, np.newaxis], 3, axis=2)\n\n            X[j] = eeg_data_expanded\n            if self.mode != 'test':\n                y[j] = row[CLASSES]\n\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.898131Z","iopub.execute_input":"2024-03-30T05:42:50.898712Z","iopub.status.idle":"2024-03-30T05:42:50.913015Z","shell.execute_reply.started":"2024-03-30T05:42:50.898683Z","shell.execute_reply":"2024-03-30T05:42:50.912199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def lrfn(epoch):\n    lr_schedule = [1e-3, 1e-3, 1e-3, 1e-4, 1e-4, 1e-4, 1e-5, 1e-5, 1e-5]\n    return lr_schedule[epoch]\n\n# Define the learning rate scheduler callback\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nimport efficientnet.tfkeras as efn\n\ndef build_model():\n\n    # inp = tf.keras.Input(shape=(128,256,8))\n    inp = tf.keras.Input(shape=(1536, 512, 3))\n\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n\n    # # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # # KAGGLE SPECTROGRAMS\n    # x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    # x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n    # # EEG SPECTROGRAMS\n    # x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    # x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n    # # MAKE 512X512X3\n    # if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n    #     x = tf.keras.layers.Concatenate(axis=2)([x1,x2])\n    # elif USE_EEG_SPECTROGRAMS: x = x2\n    # else: x = x1\n    # x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n\n    # OUTPUT\n    x = base_model(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n\n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.914289Z","iopub.execute_input":"2024-03-30T05:42:50.914623Z","iopub.status.idle":"2024-03-30T05:42:50.93063Z","shell.execute_reply.started":"2024-03-30T05:42:50.914595Z","shell.execute_reply":"2024-03-30T05:42:50.929981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## cross_validate_model - Label Refine","metadata":{}},{"cell_type":"code","source":"def cross_validate_model(train_data, train_data_2, folds, random_seed, targets, nome_modelo):\n    inicio = time.time()\n    path_model = f'MLP_Model{nome_modelo}'\n    if not os.path.exists(path_model):\n        os.makedirs(path_model)\n\n    all_oof = []\n    all_oof2 = []\n    all_true = []\n    models = []\n    score_list = []\n    \n    # Separating the data to iterate over both dataframes simultaneously\n    gkf = StratifiedGroupKFold(n_splits=folds, shuffle=True, random_state=random_seed)\n    splits1 = list(gkf.split(train_data, train_data[[\"expert_consensus\"]], train_data[\"patient_id\"]))\n    splits2 = list(gkf.split(train_data_2, train_data_2[[\"expert_consensus\"]], train_data_2[\"patient_id\"]))\n\n    # Iterate over folds in parallel\n    for i, ((train_index, valid_index), (train_index2, valid_index2)) in enumerate(zip(splits1, splits2)):\n        \n        # Copy the dataframes to avoid leaks\n        train_data_ = train_data.copy()\n        train_data_2_ = train_data_2.copy()\n        set_random_seed(random_seed, deterministic=True)\n        \n        # Start folding\n        print('#' * 25)\n        print(f'### Fold {i + 1}')\n        print(f'### train size 1 {len(train_index)}, valid size {len(valid_index)}')\n        print(f'### train size 2 {len(train_index2)}, valid size {len(valid_index2)}')\n        print('#' * 25)\n\n        ### --------------------------- Performs model 1 training -------------- --------------------------- ###\n        K.clear_session()\n        train_gen = DataGenerator(train_data_.iloc[train_index], shuffle=True, batch_size=BATCH)\n        valid_gen = DataGenerator(train_data_.iloc[valid_index], shuffle=False, batch_size=(BATCH*2), mode='valid')\n        model = build_EfficientNetB0(input_shape=(512, 512, 3), num_classes=6)\n        history = model.fit(train_gen, verbose=2, validation_data=valid_gen, epochs=EPOCHS, callbacks=[LR])\n\n        # Model training result 1\n        train_loss = history.history['loss'][-1]  \n        valid_loss = history.history['val_loss'][-1]\n        print(f'train_loss 1 {train_loss} valid_loss 1 {valid_loss}')\n        score_list.append((train_loss, valid_loss))\n\n        \n        ### --------------------------- creation of pseudo labels ---------------- ------------------------- ###\n        # pseudo labels for low quality data\n        train_2_index_total_gen = DataGenerator(train_data_2_.iloc[train_index2], shuffle=False, batch_size=BATCH)\n        pseudo_labels_2 = model.predict(train_2_index_total_gen, verbose=2)\n        # Refinement of low quality labels\n        train_data_2_.loc[train_index2, TARGETS] /= 2\n        train_data_2_.loc[train_index2, TARGETS] += pseudo_labels_2 / 2\n\n        # pseudo labels for high quality data (50% of data)\n        train_data_3_ = train_data_\n        train_3_index_total_gen = DataGenerator(train_data_3_.iloc[train_index], shuffle=False, batch_size=BATCH)\n        pseudo_labels_3 = model.predict(train_3_index_total_gen, verbose=2)\n        # Refinement of high quality labels\n        train_data_3_.loc[train_index, TARGETS] /= 2\n        train_data_3_.loc[train_index, TARGETS] += pseudo_labels_3 / 2\n\n        ### --------------------------- Creation of the data generator for the refined labels model --------- -------------------------------- ###\n        # Low quality data\n        np.random.shuffle(train_index)\n        np.random.shuffle(valid_index)\n        sixty_percent_length = int(0.5 * len(train_data_3_))\n        train_index_60 = train_index[:int(sixty_percent_length * len(train_index) / len(train_data_3_))]\n        valid_index_60 = valid_index[:int(sixty_percent_length * len(valid_index) / len(train_data_3_))]\n        train_gen_2 = DataGenerator(pd.concat([train_data_3_.iloc[train_index_60], train_data_2_.iloc[train_index2]]), shuffle=True, batch_size=BATCH)\n        valid_gen_2 = DataGenerator(pd.concat([train_data_3_.iloc[valid_index_60], train_data_2_.iloc[valid_index2]]), shuffle=False, batch_size=BATCH*2, mode='valid')\n        # Rebuild the high quality data generator with 50% of the labels refined\n        train_gen = DataGenerator(train_data_.iloc[train_index], shuffle=True, batch_size=BATCH)\n        valid_gen = DataGenerator(train_data_.iloc[valid_index], shuffle=False, batch_size=(BATCH*2), mode='valid')\n        \n        ### --------------------------- Model 2 training and finetunning -------------- --------------------------- ###\n        K.clear_session()\n        new_model = build_EfficientNetB0(input_shape=(512, 512, 3), num_classes=6)\n        # Training with the refined low-quality data\n        history = new_model.fit(train_gen_2, verbose=2, validation_data=valid_gen_2, epochs=EPOCHS, callbacks=[LR])\n        # Finetuning with refined high-quality data\n        history = new_model.fit(train_gen, verbose=2, validation_data=valid_gen, epochs=EPOCHS, callbacks=[LR])\n        new_model.save_weights(f'{path_model}/MLP_fold{i}.weights.h5')\n        models.append(new_model)\n\n        # Model 2 training result\n        train_loss = history.history['loss'][-1]  # Valor da perda do último epoch de treinamento\n        valid_loss = history.history['val_loss'][-1]  # Valor da perda do último epoch de validação\n        print(f'train_loss 2 {train_loss} valid_loss 2 {valid_loss}')\n        score_list.append((train_loss, valid_loss))\n\n\n        # MLP OOF\n        oof = new_model.predict(valid_gen, verbose=2)\n        all_oof.append(oof)\n        all_true.append(train_data.iloc[valid_index][TARGETS].values)\n\n        # TRAIN MEAN OOF\n        y_train = train_data.iloc[train_index][targets].values\n        y_valid = train_data.iloc[valid_index][targets].values\n        oof = y_valid.copy()\n        for j in range(6):\n            oof[:,j] = y_train[:,j].mean()\n        oof = oof / oof.sum(axis=1,keepdims=True)\n        all_oof2.append(oof)\n\n        del model, new_model, train_gen, valid_gen, train_2_index_total_gen, train_gen_2, valid_gen_2, oof, y_train, y_valid, train_index, valid_index\n        K.clear_session()\n        gc.collect()\n\n        if i==folds-1: break\n\n    all_oof = np.concatenate(all_oof)\n    all_oof2 = np.concatenate(all_oof2)\n    all_true = np.concatenate(all_true)\n\n    oof = pd.DataFrame(all_oof.copy())\n    oof['id'] = np.arange(len(oof))\n\n    true = pd.DataFrame(all_true.copy())\n    true['id'] = np.arange(len(true))\n\n    cv = score(solution=true, submission=oof, row_id_column_name='id')\n    fim = time.time()\n    tempo_execucao = fim - inicio\n    print(f'{nome_modelo} CV Score with EEG Spectrograms ={cv} tempo: {tempo_execucao}')\n    \n    gc.collect()\n\n    score_array = np.array(score_list)\n    std_dev = np.std(score_array, axis=0)\n    std_dev = std_dev.tolist()\n\n    return cv, tempo_execucao, all_oof, all_oof2, all_true, models, score_list, std_dev, path_model","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.933931Z","iopub.execute_input":"2024-03-30T05:42:50.93423Z","iopub.status.idle":"2024-03-30T05:42:50.962383Z","shell.execute_reply.started":"2024-03-30T05:42:50.934209Z","shell.execute_reply":"2024-03-30T05:42:50.961503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not ONLY_INFERENCE:\n    high_quality_df, low_quality_df = create_train_data()\n    result, tempo_execucao, all_oof, all_oof2, all_true, models, score_list, std_dev, path_model = cross_validate_model(high_quality_df, low_quality_df, FOLDS, 42, CLASSES, NAME)\n    print(f'Result cv V1 final {result}{tempo_execucao} {score_list} {std_dev}')\n    display(result)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.963431Z","iopub.execute_input":"2024-03-30T05:42:50.963772Z","iopub.status.idle":"2024-03-30T05:42:50.979573Z","shell.execute_reply.started":"2024-03-30T05:42:50.963743Z","shell.execute_reply":"2024-03-30T05:42:50.978891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# INFERENCE","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:50.980637Z","iopub.execute_input":"2024-03-30T05:42:50.981172Z","iopub.status.idle":"2024-03-30T05:42:51.019196Z","shell.execute_reply.started":"2024-03-30T05:42:50.981142Z","shell.execute_reply":"2024-03-30T05:42:51.018381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:51.02016Z","iopub.execute_input":"2024-03-30T05:42:51.020406Z","iopub.status.idle":"2024-03-30T05:42:51.359993Z","shell.execute_reply.started":"2024-03-30T05:42:51.020385Z","shell.execute_reply":"2024-03-30T05:42:51.359014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:51.364186Z","iopub.execute_input":"2024-03-30T05:42:51.364589Z","iopub.status.idle":"2024-03-30T05:42:51.511528Z","shell.execute_reply.started":"2024-03-30T05:42:51.364549Z","shell.execute_reply":"2024-03-30T05:42:51.510739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:42:51.512468Z","iopub.execute_input":"2024-03-30T05:42:51.513046Z","iopub.status.idle":"2024-03-30T05:43:07.204429Z","shell.execute_reply.started":"2024-03-30T05:42:51.51302Z","shell.execute_reply":"2024-03-30T05:43:07.203486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_spectrogram_with_cusignal(eeg_data, eeg_id, start, duration= 50,\n                                    low_cut_freq = 0.7, high_cut_freq = 20, order_band = 5,\n                                    spec_size_freq = 267, spec_size_time = 30,\n                                    nperseg_ = 1500, noverlap_ = 1483, nfft_ = 2750,\n                                    sigma_gaussian = 0.7, \n                                    mean_montage_names = 4):\n    \n    electrode_names = ['LL', 'RL', 'LP', 'RP']\n\n    electrode_pairs = [\n        ['Fp1', 'F7', 'T3', 'T5', 'O1'],\n        ['Fp2', 'F8', 'T4', 'T6', 'O2'],\n        ['Fp1', 'F3', 'C3', 'P3', 'O1'],\n        ['Fp2', 'F4', 'C4', 'P4', 'O2']\n    ]\n    \n    # Filter specifications\n    nyquist_freq = 0.5 * 200\n    low_cut_freq_normalized = low_cut_freq / nyquist_freq\n    high_cut_freq_normalized = high_cut_freq / nyquist_freq\n\n    # Bandpass and notch filter\n    bandpass_coefficients = butter(order_band, [low_cut_freq_normalized, high_cut_freq_normalized], btype='band')\n    notch_coefficients = iirnotch(w0=60, Q=30, fs=200)\n    \n    spec_size = duration * 200\n    start = start * 200\n    real_start = start + (10_000//2) - (spec_size//2)\n    eeg_data = eeg_data.iloc[real_start:real_start+spec_size]\n    \n    \n    # Spectrogram parameters\n    fs = 200\n    nperseg = nperseg_\n    noverlap = noverlap_\n    nfft = nfft_\n    \n    if spec_size_freq <=0 or spec_size_time <=0:\n        frequencias_size = int((nfft // 2)/5.15198)+1\n        segmentos = int((spec_size - noverlap) / (nperseg - noverlap)) \n    else:\n        frequencias_size = spec_size_freq\n        segmentos = spec_size_time\n        \n    spectrogram = cp.zeros((frequencias_size, segmentos, 4), dtype='float32')\n    \n    processed_eeg = {}\n\n    for i, name in enumerate(electrode_names):\n        cols = electrode_pairs[i]\n        processed_eeg[name] = np.zeros(spec_size)\n        for j in range(4):\n            # Compute differential signals\n            signal = cp.array(eeg_data[cols[j]].values - eeg_data[cols[j+1]].values)\n\n            # Handle NaNs\n            mean_signal = cp.nanmean(signal)\n            signal = cp.nan_to_num(signal, nan=mean_signal) if cp.isnan(signal).mean() < 1 else cp.zeros_like(signal)\n            \n\n            # Filter bandpass and notch\n            signal_filtered = filtfilt(*notch_coefficients, signal.get())\n            signal_filtered = filtfilt(*bandpass_coefficients, signal_filtered)\n            signal = cp.asarray(signal_filtered)\n            \n            frequencies, times, Sxx = cusignal.spectrogram(signal, fs, nperseg=nperseg, noverlap=noverlap, nfft=nfft)\n\n            # Filter frequency range\n            valid_freqs = (frequencies >= 0.59) & (frequencies <= 20)\n            frequencies_filtered = frequencies[valid_freqs]\n            Sxx_filtered = Sxx[valid_freqs, :]\n\n            # Logarithmic transformation and normalization using Cupy\n            spectrogram_slice = cp.clip(Sxx_filtered, cp.exp(-4), cp.exp(6))\n            spectrogram_slice = cp.log10(spectrogram_slice)\n\n            normalization_epsilon = 1e-6\n            mean = spectrogram_slice.mean(axis=(0, 1), keepdims=True)\n            std = spectrogram_slice.std(axis=(0, 1), keepdims=True)\n            spectrogram_slice = (spectrogram_slice - mean) / (std + normalization_epsilon)\n            \n            spectrogram[:, :, i] += spectrogram_slice\n            processed_eeg[f'{cols[j]}_{cols[j+1]}'] = signal.get()\n            processed_eeg[name] += signal.get()\n        \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        if mean_montage_names > 0:\n            spectrogram[:,:,i] /= mean_montage_names\n\n    # Convert to NumPy and apply Gaussian filter\n    spectrogram_np = cp.asnumpy(spectrogram)\n    if sigma_gaussian > 0.0:\n        spectrogram_np = gaussian_filter(spectrogram_np, sigma=sigma_gaussian)\n\n    # Filter EKG signal\n    ekg_signal_filtered = filtfilt(*notch_coefficients, eeg_data[\"EKG\"].values)\n    ekg_signal_filtered = filtfilt(*bandpass_coefficients, ekg_signal_filtered)\n    processed_eeg['EKG'] = np.array(ekg_signal_filtered)\n\n    return spectrogram_np, processed_eeg","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:43:07.205873Z","iopub.execute_input":"2024-03-30T05:43:07.206415Z","iopub.status.idle":"2024-03-30T05:43:07.226208Z","shell.execute_reply.started":"2024-03-30T05:43:07.20639Z","shell.execute_reply":"2024-03-30T05:43:07.225303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_spectogram_competition(spec_id, seconds_min):\n    spec = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/{spec_id}.parquet')\n    inicio = (seconds_min) // 2\n    img = spec.fillna(0).values[:, 1:].T.astype(\"float32\")\n    img = img[:, inicio:inicio+300]\n    \n    # Log transform and normalize\n    img = np.clip(img, np.exp(-4), np.exp(6))\n    img = np.log(img)\n    eps = 1e-6\n    img_mean = img.mean()\n    img_std = img.std()\n    img = (img - img_mean) / (img_std + eps)\n    \n    return img ","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:43:07.227586Z","iopub.execute_input":"2024-03-30T05:43:07.227928Z","iopub.status.idle":"2024-03-30T05:43:07.25102Z","shell.execute_reply.started":"2024-03-30T05:43:07.227896Z","shell.execute_reply":"2024-03-30T05:43:07.250256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom tqdm import tqdm\nimport pandas as pd\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\nall_eegs2 = {}\n# Make sure the 'images' folder exists\noutput_folder = 'imagens'\nif not os.path.exists(output_folder):\n    os.makedirs(output_folder)\n    \ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape:',test.shape)\nprint(test.head())\n\n# Creation of spectograms on the test base\nfor i in tqdm(range(len(test)), desc=\"Processing EEGs\"):\n    row = test.iloc[i]\n    eeg_id = row['eeg_id']\n    spec_id = row['spectrogram_id']\n    seconds_min = 0\n    start_second = 0\n    eeg_data = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/{eeg_id}.parquet')\n    eeg_new_key = eeg_id\n    image_50s, _ = create_spectrogram_with_cusignal(eeg_data=eeg_data, eeg_id=eeg_id, start=start_second, duration= 50,\n                                    low_cut_freq = 0.7, high_cut_freq = 20, order_band = 5,\n                                    spec_size_freq = 267, spec_size_time = 501,\n                                    nperseg_ = 1500, noverlap_ = 1483, nfft_ = 2750,\n                                    sigma_gaussian = 0.0, \n                                    mean_montage_names = 4)\n    image_10s, _ = create_spectrogram_with_cusignal(eeg_data=eeg_data, eeg_id=eeg_id, start=start_second, duration= 10,\n                                    low_cut_freq = 0.7, high_cut_freq = 20, order_band = 5,\n                                    spec_size_freq = 100, spec_size_time = 291,\n                                    nperseg_ = 260, noverlap_ = 254, nfft_ = 1030,\n                                    sigma_gaussian = 0.0, \n                                    mean_montage_names = 4)\n    image_10m = create_spectogram_competition(spec_id, seconds_min)\n    \n    imagem_final_unico_canal = np.zeros((1068, 501))\n    for j in range(4):\n        inicio = j * 267 \n        fim = inicio + 267\n        imagem_final_unico_canal[inicio:fim, :] = image_50s[:, :, j]\n        \n    \n    imagem_final_unico_canal2 = np.zeros((400, 291))\n    for n in range(4):\n        inicio = n * 100 \n        fim = inicio + 100\n        imagem_final_unico_canal2[inicio:fim, :] = image_10s[:, :, n]\n    \n    imagem_final_unico_canal_resized = cv2.resize(imagem_final_unico_canal, (400, 800), interpolation=cv2.INTER_AREA)\n    imagem_final_unico_canal2_resized = cv2.resize(imagem_final_unico_canal2, (300, 400), interpolation=cv2.INTER_AREA)\n    eeg_new_resized = cv2.resize(image_10m, (300, 400), interpolation=cv2.INTER_AREA)\n    imagem_final = np.zeros((800, 700), dtype=np.float32)\n    imagem_final[0:800, 0:400] = imagem_final_unico_canal_resized\n    imagem_final[0:400,400:700] = imagem_final_unico_canal2_resized\n    imagem_final[400:800, 400:700] = eeg_new_resized\n    imagem_final = imagem_final[::-1]\n    \n    imagem_final = cv2.resize(imagem_final, (512, 512), interpolation=cv2.INTER_AREA)\n    \n    all_eegs2[eeg_new_key] = imagem_final\n    \n    if i ==0:\n        plt.figure(figsize=(10, 10))\n        plt.imshow(imagem_final, cmap='jet')\n        plt.axis('off')\n        plt.show()\n\n        print(imagem_final.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:43:07.252178Z","iopub.execute_input":"2024-03-30T05:43:07.252454Z","iopub.status.idle":"2024-03-30T05:44:54.095283Z","shell.execute_reply.started":"2024-03-30T05:43:07.25243Z","shell.execute_reply":"2024-03-30T05:44:54.094354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nclass DataGeneratorTest(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, specs, eeg_specs, batch_size=32, shuffle=False, eegs={}, mode='train'):\n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.eegs = eegs\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n\n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples'\n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n\n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test':\n                r = 0\n            else:\n                r = int( (row['spectrogram_label_offset_seconds_min'] + row['spectrogram_label_offset_seconds_max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spectrogram_id][r:r+300,k*100:(k+1)*100].T\n\n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n\n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n\n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n\n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n\n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n        \n        # EEG信号\n        x1 = [X[:,:,:,i:i+1] for i in range(4)]\n        x1 = np.concatenate(x1, axis=1)\n\n        # EEG频谱\n        x2 = [X[:,:,:,i+4:i+5] for i in range(4)]\n        x2 = np.concatenate(x2, axis=1)\n\n        X = np.concatenate([x1, x2], axis=2)\n        \n#         X = np.zeros((len(indexes),SPEC_SIZE[0],SPEC_SIZE[1],SPEC_SIZE[2]),dtype='float32')\n        X2 = np.zeros((len(indexes),SPEC_SIZE[0],SPEC_SIZE[1],1),dtype='float32')\n\n        y = np.zeros((len(indexes),6),dtype='float32')\n\n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            eeg_data = self.eegs[row.eeg_id] \n            eeg_data_expanded = eeg_data[:, :, np.newaxis]\n#             eeg_data_expanded = np.repeat(eeg_data[:, :, np.newaxis], 3, axis=2)\n            X2[j,] = eeg_data_expanded\n            if self.mode!='test':\n                y[j] = row[CLASSES]\n                \n        X = np.concatenate([X, X2, X2], axis=1)\n        X = np.repeat(X, 3, axis=3)\n        \n        return X,y","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:44:54.096622Z","iopub.execute_input":"2024-03-30T05:44:54.096915Z","iopub.status.idle":"2024-03-30T05:44:54.118069Z","shell.execute_reply.started":"2024-03-30T05:44:54.096891Z","shell.execute_reply":"2024-03-30T05:44:54.117028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER MLP ON TEST\npreds = []\nmodel = build_model()\ntest_gen = DataGeneratorTest(test, specs=spectrograms2, eeg_specs=all_eegs, shuffle=False, batch_size=1, eegs=all_eegs2, mode='test')\n\nprint('Inferring test... ',end='')\nfor i in range(FOLDS):\n    print(f'fold {i+1}, ',end='')\n    model.load_weights(f'/kaggle/input/mactf-v2/TF5/EffNet_v5_f{i}.h5')\n    pred = model.predict(test_gen, verbose=0)\n    preds.append(pred)\npred_origin = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred_origin.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:44:54.119505Z","iopub.execute_input":"2024-03-30T05:44:54.120075Z","iopub.status.idle":"2024-03-30T05:45:04.280306Z","shell.execute_reply.started":"2024-03-30T05:44:54.120042Z","shell.execute_reply":"2024-03-30T05:45:04.279396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import os, gc\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\nimport tensorflow as tf\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nprint('TensorFlow version =',tf.__version__)\n\n# USE MULTIPLE GPUS\ngpus = tf.config.list_physical_devices('GPU')\nif len(gpus)<=1: \n    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n    print(f'Using {len(gpus)} GPU')\nelse: \n    strategy = tf.distribute.MirroredStrategy()\n    print(f'Using {len(gpus)} GPUs')\n\nVER = 5\n\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = '/kaggle/input/brain-efficientnet-models-v3-v4-v5/'\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE MIXED PRECISION\nMIX = True\nif MIX:\n    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n    print('Mixed precision enabled')\nelse:\n    print('Using full precision')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as albu\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x:y for y,x in TARS.items()}\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, data, batch_size=32, shuffle=False, augment=False, mode='train',\n                 specs = None, eeg_specs = None): \n\n        self.data = data\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        self.on_epoch_end()\n        \n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n        return ct\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        X, y = self.__data_generation(indexes)\n        if self.augment: X = self.__augment_batch(X) \n        return X, y\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange( len(self.data) )\n        if self.shuffle: np.random.shuffle(self.indexes)\n                        \n    def __data_generation(self, indexes):\n        'Generates data containing batch_size samples' \n        \n        X = np.zeros((len(indexes),128,256,8),dtype='float32')\n        y = np.zeros((len(indexes),6),dtype='float32')\n        img = np.ones((128,256),dtype='float32')\n        \n        for j,i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode=='test': \n                r = 0\n            else: \n                r = int( (row['min'] + row['max'])//4 )\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300,k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img,np.exp(-4),np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img-m)/(s+ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j,14:-14,:,k] = img[:,22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j,:,:,4:] = img\n                \n            if self.mode!='test':\n                y[j,] = row[TARGETS]\n            \n        return X,y\n    \n    def __random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            #albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment_batch(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        return img_batch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gen = DataGenerator(train, batch_size=32, shuffle=False)\n# ROWS=2; COLS=3; BATCHES=2\n\n# for i,(x,y) in enumerate(gen):\n#     plt.figure(figsize=(20,8))\n#     for j in range(ROWS):\n#         for k in range(COLS):\n#             plt.subplot(ROWS,COLS,j*COLS+k+1)\n#             t = y[j*COLS+k]\n#             img = x[j*COLS+k,:,:,0][::-1,]\n#             mn = img.flatten().min()\n#             mx = img.flatten().max()\n#             img = (img-mn)/(mx-mn)\n#             plt.imshow(img)\n#             tars = f'[{t[0]:0.2f}'\n#             for s in t[1:]: tars += f', {s:0.2f}'\n#             eeg = train.eeg_id.values[i*32+j*COLS+k]\n#             plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n#             plt.yticks([])\n#             plt.ylabel('Frequencies (Hz)',size=14)\n#             plt.xlabel('Time (sec)',size=16)\n#     plt.show()\n#     if i==BATCHES-1: break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import math\n# LR_START = 1e-6\n# LR_MAX = 1e-3\n# LR_MIN = 1e-6\n# LR_RAMPUP_EPOCHS = 0\n# LR_SUSTAIN_EPOCHS = 0\n# EPOCHS2 = 10\n\n# def lrfn(epoch):\n#     if epoch < LR_RAMPUP_EPOCHS:\n#         lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n#     elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n#         lr = LR_MAX\n#     else:\n#         decay_total_epochs = EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n#         decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n#         phase = math.pi * decay_epoch_index / decay_total_epochs\n#         cosine_decay = 0.5 * (1 + math.cos(phase))\n#         lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n#     return lr\n\n# rng = [i for i in range(EPOCHS2)]\n# lr_y = [lrfn(x) for x in rng]\n# plt.figure(figsize=(10, 4))\n# plt.plot(rng, lr_y, '-o')\n# plt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\n# plt.title('Cosine Training Schedule',size=16); plt.show()\n\n# LR2 = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LR_START = 1e-4\n# LR_MAX = 1e-3\n# LR_RAMPUP_EPOCHS = 0\n# LR_SUSTAIN_EPOCHS = 1\n# LR_STEP_DECAY = 0.1\n# EVERY = 1\n# EPOCHS = 4\n\n# def lrfn(epoch):\n#     if epoch < LR_RAMPUP_EPOCHS:\n#         lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n#     elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n#         lr = LR_MAX\n#     else:\n#         lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//EVERY)\n#     return lr\n\n# rng = [i for i in range(EPOCHS)]\n# y = [lrfn(x) for x in rng]\n# plt.figure(figsize=(10, 4))\n# plt.plot(rng, y, 'o-'); \n# plt.xlabel('epoch',size=14); plt.ylabel('learning rate',size=14)\n# plt.title('Step Training Schedule',size=16); plt.show()\n\n# LR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/tf-efficientnet-whl-files /kaggle/input/tf-efficientnet-whl-files/efficientnet-1.1.1-py3-none-any.whl","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\n\ndef build_model():\n    \n    inp = tf.keras.Input(shape=(128,256,8))\n    base_model = efn.EfficientNetB0(include_top=False, weights=None, input_shape=None)\n    base_model.load_weights('/kaggle/input/tf-efficientnet-imagenet-weights/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5')\n    \n    # RESHAPE INPUT 128x256x8 => 512x512x3 MONOTONE IMAGE\n    # KAGGLE SPECTROGRAMS\n    x1 = [inp[:,:,:,i:i+1] for i in range(4)]\n    x1 = tf.keras.layers.Concatenate(axis=1)(x1)\n    # EEG SPECTROGRAMS\n    x2 = [inp[:,:,:,i+4:i+5] for i in range(4)]\n    x2 = tf.keras.layers.Concatenate(axis=1)(x2)\n    # MAKE 512X512X3\n    if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n        x = tf.keras.layers.Concatenate(axis=2)([x1,x2])\n    elif USE_EEG_SPECTROGRAMS: x = x2\n    else: x = x1\n    x = tf.keras.layers.Concatenate(axis=3)([x,x,x])\n    \n    # OUTPUT\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n        \n    # COMPILE MODEL\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n    loss = tf.keras.losses.KLDivergence()\n\n    model.compile(loss=loss, optimizer = opt) \n        \n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import KFold, GroupKFold\n# import tensorflow.keras.backend as K, gc\n\n# all_oof = []\n# all_true = []\n\n# gkf = GroupKFold(n_splits=5)\n# for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    \n#     print('#'*25)\n#     print(f'### Fold {i+1}')\n    \n#     train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=32, augment=False)\n#     valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n    \n#     print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n#     print('#'*25)\n    \n#     K.clear_session()\n#     with strategy.scope():\n#         model = build_model()\n#     if LOAD_MODELS_FROM is None:\n#         model.fit(train_gen, verbose=1,\n#               validation_data = valid_gen,\n#               epochs=EPOCHS, callbacks = [LR])\n#         model.save_weights(f'EffNet_v{VER}_f{i}.h5')\n#     else:\n#         model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n        \n#     oof = model.predict(valid_gen, verbose=1)\n#     all_oof.append(oof)\n#     all_true.append(train.iloc[valid_index][TARGETS].values)\n    \n#     del model, oof\n#     gc.collect()\n    \n# all_oof = np.concatenate(all_oof)\n# all_true = np.concatenate(all_true)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import sys\n# sys.path.append('/kaggle/input/kaggle-kl-div')\n# from kaggle_kl_div import score\n\n# oof = pd.DataFrame(all_oof.copy())\n# oof['id'] = np.arange(len(oof))\n\n# true = pd.DataFrame(all_true.copy())\n# true['id'] = np.arange(len(true))\n\n# cv = score(solution=true, submission=oof, row_id_column_name='id')\n# print('CV Score KL-Div for EfficientNetB2 =',cv)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i,f in enumerate(files2):\n    if i%100==0: print(i,', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:,1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = img","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER EFFICIENTNET ON TEST\npreds = []\nmodel = build_model()\ntest_gen = DataGenerator(test, shuffle=False, batch_size=1, mode='test',\n                         specs = spectrograms2, eeg_specs = all_eegs2)\n\nfor i in range(5):\n    print(f'Fold {i+1}')\n    if LOAD_MODELS_FROM:\n        model.load_weights(f'{LOAD_MODELS_FROM}EffNet_v{VER}_f{i}.h5')\n    else:\n        model.load_weights(f'EffNet_v{VER}_f{i}.h5')\n    pred = model.predict(test_gen, verbose=1)\n    preds.append(pred)\npred_c = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred_c.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport os\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, BackboneFinetuning, EarlyStopping\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn import model_selection\nimport torchvision.transforms as transforms\nimport torchvision.io \nimport librosa\nfrom PIL import Image\nimport albumentations as alb\nimport torch.multiprocessing as mp\nimport warnings\nimport os\n\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nwarnings.filterwarnings('ignore')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Config:\n    \n    use_aug = False\n    num_classes = 6\n    batch_size = 1\n    epochs = 18\n    PRECISION = 16    \n    PATIENCE = 10    \n    seed = 2024\n    pretrained = False            \n    weight_decay = 1e-3\n    use_mixup = False\n    mixup_alpha = 0.02   \n    num_channels = 8\n    backbone_2d = 'tf_efficientnet_b0'\n    data_root = \"/kaggle/input/hms-harmful-brain-activity-classification/\"\n    kaggle_spec_path = \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\" \n    TRAIN_EEGS = \"/home/nischay//brain/Data/spec/EEG_Spectrograms/\"\n    LR = 7e-3\n    processed_train = None\n    model_dir = '/kaggle/input/mulitimod-exp1-5gkf-b08ch-cv606'\n    trn_folds = [0,1,2,3,4]\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS2 = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n\n\ndef maddest(d, axis: int = None):\n    \"\"\"\n    Denoise function.\n    \"\"\"\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x: np.ndarray, wavelet: str = 'haar', level: int = 1): \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\") # multilevel 1D Discrete Wavelet Transform of data.\n    sigma = (1/0.6745) * maddest(coeff[-level])\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n    output = pywt.waverec(coeff, wavelet, mode='per')\n    return output\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display:\n        plt.figure(figsize=(10,7))\n        \n    signals = []\n    \n    for k in range(4):\n        COLS = FEATS2[k]\n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img\n\ndef plot_spectrogram(spectrogram_path: str):\n    \"\"\"\n    Source: https://www.kaggle.com/code/mvvppp/hms-eda-and-domain-journey\n    Visualize spectrogram recordings from a parquet file.\n    :param spectrogram_path: path to the spectrogram parquet.\n    \"\"\"\n    sample_spect = pd.read_parquet(spectrogram_path)\n    \n    split_spect = {\n        \"LL\": sample_spect.filter(regex='^LL', axis=1),\n        \"RL\": sample_spect.filter(regex='^RL', axis=1),\n        \"RP\": sample_spect.filter(regex='^RP', axis=1),\n        \"LP\": sample_spect.filter(regex='^LP', axis=1),\n    }\n    \n    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 12))\n    axes = axes.flatten()\n    label_interval = 5\n    for i, split_name in enumerate(split_spect.keys()):\n        ax = axes[i]\n        img = ax.imshow(np.log(split_spect[split_name]).T, cmap='viridis', aspect='auto', origin='lower')\n        cbar = fig.colorbar(img, ax=ax)\n        cbar.set_label('Log(Value)')\n        ax.set_title(split_name)\n        ax.set_ylabel(\"Frequency (Hz)\")\n        ax.set_xlabel(\"Time\")\n\n        ax.set_yticks(np.arange(len(split_spect[split_name].columns)))\n        ax.set_yticklabels([column_name[3:] for column_name in split_spect[split_name].columns])\n        frequencies = [column_name[3:] for column_name in split_spect[split_name].columns]\n        ax.set_yticks(np.arange(0, len(split_spect[split_name].columns), label_interval))\n        ax.set_yticklabels(frequencies[::label_interval])\n    plt.tight_layout()\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eeg_from_parquet(parquet_path, display=False):\n    \n    # EXTRACT MIDDLE 50 SECONDS\n    eeg = pd.read_parquet(parquet_path, columns=FEATS)\n    rows = len(eeg)\n    offset = (rows-10_000)//2\n    eeg = eeg.iloc[offset:offset+10_000]\n    \n    if display: \n        plt.figure(figsize=(10,5))\n        offset = 0\n    \n    # CONVERT TO NUMPY\n    data = np.zeros((10_000,len(FEATS)))\n    for j,col in enumerate(FEATS):\n        \n        # FILL NAN\n        x = eeg[col].values.astype('float32')\n        m = np.nanmean(x)\n        if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n        else: x[:] = 0\n            \n        data[:,j] = x\n        \n        if display: \n            if j!=0: offset += x.max()\n            plt.plot(range(10_000),x-offset,label=col)\n            offset -= x.min()\n            \n    if display:\n        plt.legend()\n        name = parquet_path.split('/')[-1]\n        name = name.split('.')[0]\n        plt.title(f'EEG {name}',size=16)\n        plt.show()\n        \n    return data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape:',test.shape)\ntest.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_eegs2 = {}\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\n\nFEATS = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\n# FEATS = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2', 'EKG']\nprint(f'There are {len(FEATS)} raw eeg features')\nprint( list(FEATS) )\n\nprint('Processing Test EEG parquets...'); print()\nfor i,eeg_id in enumerate(EEG_IDS2):\n        \n    # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n    data = eeg_from_parquet(f'{PATH2}{eeg_id}.parquet', i<DISPLAY)\n    all_eegs2[eeg_id] = data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.seed_everything(Config.seed, workers=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def config_to_dict(cfg):\n    return dict((name, getattr(cfg, name)) for name in dir(cfg) if not name.startswith('__'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.DataFrame(EEG_IDS2, columns=['eeg_id'])\ntest2 = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_spec_ids = test2[test2['eeg_id'].isin(EEG_IDS2)].spectrogram_id.unique()\ntest['spectrogram_id'] = test_spec_ids","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(f'{Config.data_root}train.csv')\nprint( df.shape )\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nEEG_IDS = df.eeg_id.unique()\n\nTARGETS = df.columns[-6:]\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS_INV = {x:y for y,x in TARS.items()}\n\ntrain_df = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg({\n    'spectrogram_id':'first',\n    'spectrogram_label_offset_seconds':'min'\n})\ntrain_df.columns = ['spectogram_id','min']\n\naux = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg({\n    'spectrogram_label_offset_seconds':'max'\n})\ntrain_df['max'] = aux\n\naux = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain_df['patient_id'] = aux\n\naux = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor label in TARGETS:\n    train_df[label] = aux[label].values\n    \ny_data = train_df[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain_df[TARGETS] = y_data\n\naux = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain_df['target'] = aux\n\ntrain = train_df.reset_index()\nprint('Train non-overlapp eeg_id shape:', train_df.shape )\ntrain.head()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Config.num_classes = len(TARS.keys())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ntqdm.pandas()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom glob import glob\n\nREAD_SPEC_FILES = True\n\npaths_spectograms = glob(Config.kaggle_spec_path + \"*.parquet\")\nprint(f'There are {len(paths_spectograms)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    all_spectrograms = {}\n    for file_path in tqdm(paths_spectograms):\n        aux = pd.read_parquet(file_path)\n        name = int(file_path.split(\"/\")[-1].split('.')[0])\n        all_spectrograms[name] = aux.iloc[:,1:].values\n        del aux\nelse:\n    all_spectrograms = np.load(Config.PRE_LOADED_SPECTOGRAMS, allow_pickle=True).item()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npaths_eegs = glob(\"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\" + \"*.parquet\")\nprint(f'There are {len(paths_eegs)} EEG spectrograms')\nall_eegs = {}\ncounter = 0\n\nfor file_path in tqdm(paths_eegs):\n    eeg_id = file_path.split(\"/\")[-1].split(\".\")[0]\n    eeg_spectrogram = spectrogram_from_eeg(file_path, counter < 1)\n    all_eegs[int(eeg_id)] = eeg_spectrogram\n    counter += 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\nclass ResNet_1D_Block(nn.Module):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, downsampling):\n        super(ResNet_1D_Block, self).__init__()\n        self.bn1 = nn.BatchNorm1d(num_features=in_channels)\n        self.relu = nn.ReLU(inplace=False)\n        self.dropout = nn.Dropout(p=0.0, inplace=False)\n        self.conv1 = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n        self.conv2 = nn.Conv1d(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.maxpool = nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n        self.downsampling = downsampling\n\n    def forward(self, x):\n        identity = x\n\n        out = self.bn1(x)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv1(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv2(out)\n\n        out = self.maxpool(out)\n        identity = self.downsampling(x)\n\n        out += identity\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\n\nclass EEGMegaNet(nn.Module):\n\n    def __init__(self, backbone_2d,in_channels_2d, kernels, pretrained=False, in_channels=20, fixed_kernel_size=17, num_classes=6):\n        super(EEGMegaNet, self).__init__()\n        \n        self.kernels = kernels\n        self.planes = 24\n        self.parallel_conv = nn.ModuleList()\n        self.in_channels = in_channels\n\n\n        \n        self.backbone_2d = timm.create_model(\n            Config.backbone_2d,\n            pretrained=pretrained,\n            drop_rate = 0.1,\n            drop_path_rate = 0.1\n        \n        )\n        \n        self.features_2d = nn.Sequential(*list(self.backbone_2d.children())[:-2] + [nn.AdaptiveAvgPool2d(1),nn.Flatten()])\n        \n        # nn.Sequential(\n            # nn.AdaptiveAvgPool2d(1),\n\n        for i, kernel_size in enumerate(list(self.kernels)):\n            sep_conv = nn.Conv1d(in_channels=in_channels, out_channels=self.planes, kernel_size=(kernel_size),\n                               stride=1, padding=0, bias=False,)\n            self.parallel_conv.append(sep_conv)\n\n        self.bn1 = nn.BatchNorm1d(num_features=self.planes)\n        self.relu = nn.ReLU(inplace=False)\n        self.conv1 = nn.Conv1d(in_channels=self.planes, out_channels=self.planes, kernel_size=fixed_kernel_size,\n                               stride=2, padding=2, bias=False)\n        self.block = self._make_resnet_layer(kernel_size=fixed_kernel_size, stride=1, padding=fixed_kernel_size//2)\n        self.bn2 = nn.BatchNorm1d(num_features=self.planes)\n        self.avgpool = nn.AvgPool1d(kernel_size=4, stride=4, padding=2)\n        self.rnn = nn.GRU(input_size=self.in_channels, hidden_size=128, num_layers=1, bidirectional=True)\n        \n        self.fc1 = nn.Linear(in_features=1280, out_features=128)\n        self.fc2 = nn.Linear(in_features=736, out_features=128)\n        self.fc = nn.Linear(in_features=256, out_features=num_classes)\n\n        self.fc1d = nn.Linear(in_features=128, out_features=num_classes)\n        self.fc2d = nn.Linear(in_features=128, out_features=num_classes)\n        \n        \n        self.rnn1 = nn.GRU(input_size=156, hidden_size=156, num_layers=1, bidirectional=True)\n\n    def _make_resnet_layer(self, kernel_size, stride, blocks=8, padding=0):\n        layers = []\n        downsample = None\n        base_width = self.planes\n\n        for i in range(blocks):\n            downsampling = nn.Sequential(\n                    nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n                )\n            layers.append(ResNet_1D_Block(in_channels=self.planes, out_channels=self.planes, kernel_size=kernel_size,\n                                       stride=stride, padding=padding, downsampling=downsampling))\n\n        return nn.Sequential(*layers)\n\n    def _reshape_input(self, spec):\n        \"\"\"\n        Reshapes input (128, 256, 8) -> (512, 512, 3) monotone image.\n        \"\"\" \n        # === Get spectograms ===\n        spectograms = [spec[:, :, :, i:i+1] for i in range(4)]\n        spectograms = torch.cat(spectograms, dim=1)\n        \n        # === Get EEG spectograms ===\n        eegs = [spec[:, :, :, i:i+1] for i in range(4,8)]\n        eegs = torch.cat(eegs, dim=1)\n        \n        # === Reshape (512,512,3) ===\n        spec = spectograms\n            \n        spec = torch.cat([spec,spec,spec], dim=3)\n        spec = spec.permute(0, 3, 1, 2)\n        return spec\n\n    def forward(self, x, spec):\n\n        spec = self._reshape_input(spec)\n        spec = self.features_2d(spec)\n        # print(spec.shape) #2, 1280, 16, 8\n        out_sep = []\n\n        for i in range(len(self.kernels)):\n            sep = self.parallel_conv[i](x)\n            out_sep.append(sep)\n\n        out = torch.cat(out_sep, dim=2)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv1(out)  \n\n        out = self.block(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.avgpool(out)  \n\n\n        \n        out = out.reshape(out.shape[0], -1)  \n\n        rnn_out, _ = self.rnn(x.permute(0,2, 1))\n        new_rnn_h = rnn_out[:, -1, :]  \n\n        new_out = torch.cat([out, new_rnn_h], dim=1)  \n        new_out = self.fc2(new_out)  \n        out1d = self.fc1d(new_out)\n        \n        spec = self.fc1(spec)  \n        out2d = self.fc2d(spec)\n        \n        result = torch.cat([new_out, spec], dim=1)  \n        result = self.fc(result)\n        \n        \n        return result, new_out, spec, out1d, out2d","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\niot = torch.randn(2, Config.num_channels, 10000)#.cuda()\nspec = torch.randn(2, 128, 256, 8)#.cuda()\n\nmodel = EEGMegaNet(backbone_2d=Config.backbone_2d,in_channels_2d=8,kernels=[3,5,7,9],pretrained=False, in_channels=Config.num_channels, fixed_kernel_size=5, num_classes=6)#.cuda()\noutput,_,_,_,_ = model(iot, spec)\nprint(output.shape)\n\ndel iot, model\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import butter, lfilter\n\ndef quantize_data(data, classes):\n    mu_x = mu_law_encoding(data, classes)\n    # bins = np.linspace(-1, 1, classes)\n    # quantized = np.digitize(mu_x, bins) - 1\n    return mu_x#quantized\n\ndef mu_law_encoding(data, mu):\n    mu_x = np.sign(data) * np.log(1 + mu * np.abs(data)) / np.log(mu + 1)\n    return mu_x\n\ndef mu_law_expansion(data, mu):\n    s = np.sign(data) * (np.exp(np.abs(data) * np.log(mu + 1)) - 1) / mu\n    return s\n\ndef butter_lowpass_filter(data, cutoff_freq=20, sampling_rate=200, order=4):\n    nyquist = 0.5 * sampling_rate\n    normal_cutoff = cutoff_freq / nyquist\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    filtered_data = lfilter(b, a, data, axis=0)\n    return filtered_data\n\nclass EEGDataset(torch.utils.data.Dataset):\n\n    def __init__(self, data, eegs=None, augmentations = None, test = False): \n\n        self.data = data\n        self.eegs = eegs\n        self.augmentations = augmentations\n        self.test = test\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n\n        row = self.data.iloc[index]      \n        data = self.eegs[row.eeg_id]\n\n        sample = np.zeros((data.shape[0], 8))  # Assuming data.shape[1] == 8 for 8 channels\n        \n        # Mapping of channel names to indices\n        FEAT2IDX = {'Fp1': 0, 'T3': 1, 'C3': 2, 'O1': 3, 'Fp2': 4, 'C4': 5, 'T4': 6, 'O2': 7}\n        \n        # Compute differences\n        sample[:,0] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['T3']]\n        sample[:,1] = data[:,FEAT2IDX['T3']] - data[:,FEAT2IDX['O1']]\n        \n        sample[:,2] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['C3']]\n        sample[:,3] = data[:,FEAT2IDX['C3']] - data[:,FEAT2IDX['O1']]\n        \n        sample[:,4] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['C4']]\n        sample[:,5] = data[:,FEAT2IDX['C4']] - data[:,FEAT2IDX['O2']]\n        \n        sample[:,6] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['T4']]\n        sample[:,7] = data[:,FEAT2IDX['T4']] - data[:,FEAT2IDX['O2']]\n\n        # sample = np.concatenate([sample,data],1)\n        \n        # Feature Engineering on sample instead of data\n        \n        sample = (sample - np.mean(sample, axis=0)) / np.std(sample, axis=0)\n\n        sample = np.clip(sample, -1024, 1024)\n        sample = np.nan_to_num(sample, nan=0)# / 32.0\n        \n        sample = butter_lowpass_filter(sample)\n        sample = quantize_data(sample, 1)\n\n\n        samples = torch.from_numpy(sample).float()    \n        samples = samples.permute(1,0)\n        if not self.test:\n            label = row[TARGETS] \n            label = torch.tensor(label).float()  \n            return samples, label\n        else:\n            return samples\n# ================================","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nimport pandas as pd\nfrom scipy.signal import butter, lfilter\n\nclass EEGDataset(torch.utils.data.Dataset):\n    def __init__(self, data, eegs=None, specs=None, eeg_specs=None, spec_aug = False, augmentations=None, test=False): \n        self.data = data\n        self.eegs = eegs\n        self.specs = specs  # Spectrograms for each ID\n        self.eeg_specs = eeg_specs  # EEG spectrograms for each ID\n        self.augmentations = augmentations\n        self.test = test\n        self.spec_aug = spec_aug\n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        row = self.data.iloc[index]\n        \n        # Processing EEG signal data\n        eeg_data = self.eegs[row.eeg_id]\n        sample = np.zeros((eeg_data.shape[0], 8))  # Assuming eeg_data.shape[1] == 8 for 8 channels\n        FEAT2IDX = {'Fp1': 0, 'T3': 1, 'C3': 2, 'O1': 3, 'Fp2': 4, 'C4': 5, 'T4': 6, 'O2': 7}\n        \n        # Compute differences\n        for i, (start, end) in enumerate([('Fp1', 'T3'), ('T3', 'O1'), ('Fp1', 'C3'), ('C3', 'O1'), \n                                          ('Fp2', 'C4'), ('C4', 'O2'), ('Fp2', 'T4'), ('T4', 'O2')]):\n            sample[:, i] = eeg_data[:, FEAT2IDX[start]] - eeg_data[:, FEAT2IDX[end]]\n        \n        sample = self.process_sample(sample)\n        \n        samples = torch.from_numpy(sample).float()\n        samples = samples.permute(1, 0)\n        \n        # Processing spectrogram data\n        spec = self.__spec_data_generation(row)\n        if self.spec_aug:\n            spec = self.__transform(spec) \n        \n        if not self.test:\n            label = row[TARGETS]  # Assuming 'TARGETS' is defined somewhere as the label column name\n            label = torch.tensor(label).float()  \n            return samples, spec, label\n        else:\n            return samples, spec\n    \n    def process_sample(self, sample):\n        # Normalize the sample data\n        sample = (sample - np.mean(sample, axis=0)) / np.std(sample, axis=0)\n        sample = np.clip(sample, -1024, 1024)\n        sample = np.nan_to_num(sample, nan=0)\n        sample = butter_lowpass_filter(sample)\n        sample = quantize_data(sample, 1)\n        return sample\n\n    def __spec_data_generation(self, row):\n        \"\"\"\n        Generates data containing batch_size samples. This method directly\n        uses class attributes for spectrograms and EEG spectrograms.\n        \"\"\"\n        X = np.zeros((128, 256, 8), dtype='float32')   \n        \n        if not self.test:\n            # Assuming your DataFrame has a column that combines min and max values for slicing\n            r = int((row['min'] + row['max']) // 4)\n        else:\n            r = 0  # Adjust as necessary for test mode\n        \n        for region in range(4):\n            img = self.specs[row.spectrogram_id][r:r+300, region*100:(region+1)*100].T\n            \n            # Log transform spectogram\n            img = np.clip(img, np.exp(-4), np.exp(8))\n            img = np.log(img)\n\n            # Standarize per image\n            ep = 1e-6\n            mu = np.nanmean(img.flatten())\n            std = np.nanstd(img.flatten())\n            img = (img-mu)/(std+ep)\n            img = np.nan_to_num(img, nan=0.0)\n            X[14:-14, :, region] = img[:, 22:-22] / 2.0\n        \n        # Process EEG spectrogram - assuming a single channel example\n        img = self.eeg_specs[row.eeg_id]\n        X[:, :, 4:] = img                \n        return X\n\n    def __transform(self, img):\n        transforms = A.Compose([\n            A.HorizontalFlip(p=0.5),\n        ])\n        return transforms(image=img)['image']\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_dls(df_test):\n    ds_test = EEGDataset(\n        df_test, \n        eegs=all_eegs2,\n        specs=all_spectrograms,\n        eeg_specs=all_eegs,\n        augmentations = None,\n        test = True\n    )\n    dl_test = DataLoader(ds_test, batch_size=1 , shuffle=False, num_workers = 2)    \n    return dl_test, ds_test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef show_batch(img_ds, num_items, num_rows, num_cols, EEG_IDS, predict_arr=None):\n    fig = plt.figure(figsize=(12, 6))    \n    img_index = [0,0]\n    for index, img_index in enumerate(img_index):  # list first items\n        img, spec = img_ds[img_index]        \n        ax = fig.add_subplot(num_rows, num_cols, index + 1, xticks=[], yticks=[])\n        if isinstance(img, torch.Tensor):\n            img = img.detach().numpy()\n            img = img.transpose(1,0)\n        offset = 0\n        for j in range(img.shape[-1]):\n            if j != 0: offset -= img[:, j].min()\n            ax.plot(img[:, j] + offset, label=f'feature {j+1}')\n            offset += img[:, j].max() + 1  # Adding 1 for visual separation\n\n        ax.legend()\n        ax.set_title(f'EEG_Id = {EEG_IDS[img_index]}', size=14)\n\n    plt.tight_layout()\n    plt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_test = test.copy()\ndummy_test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl_test, ds_test = get_test_dls(dummy_test)\nshow_batch(ds_test, 8, 2, 4, EEG_IDS)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau, OneCycleLR\n\ndef get_optimizer(lr, params):\n    model_optimizer = torch.optim.Adam(\n            filter(lambda p: p.requires_grad, params), \n            lr=lr,\n            weight_decay=Config.weight_decay\n        )\n    interval = \"epoch\"\n    \n    lr_scheduler = CosineAnnealingWarmRestarts(\n                            model_optimizer, \n                            T_0=Config.epochs, \n                            T_mult=1, \n                            eta_min=1e-5, \n                            last_epoch=-1\n                        )\n\n    return {\n        \"optimizer\": model_optimizer, \n        \"lr_scheduler\": {\n            \"scheduler\": lr_scheduler,\n            \"interval\": interval,\n            \"monitor\": \"val_loss\",\n            \"frequency\": 1\n        }\n    }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nfrom torch.nn.functional import cross_entropy\nimport torchmetrics\nimport sklearn.metrics\nimport sys","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class KLDivLossWithLogits(nn.KLDivLoss):\n\n    def __init__(self):\n        super().__init__(reduction=\"batchmean\")\n\n    def forward(self, y, t):\n        y = nn.functional.log_softmax(y,  dim=1)\n        loss = super().forward(y, t)\n        return loss\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGModel(pl.LightningModule):\n    def __init__(self, num_classes = Config.num_classes, pretrained = Config.pretrained):\n        super().__init__()\n        self.num_classes = num_classes\n        # self.backbone = EEGNet(kernels=[3,5,7,9], in_channels=Config.num_channels, fixed_kernel_size=5, num_classes=Config.num_classes)\n        self.backbone = EEGMegaNet(backbone_2d=Config.backbone_2d,\n                                   in_channels_2d=8,\n                                   kernels=[3,5,7,9],pretrained=Config.pretrained,\n                                   in_channels=Config.num_channels,\n                                   fixed_kernel_size=5, num_classes=6)\n        \n        self.loss_function = KLDivLossWithLogits() #nn.KLDivLoss() #nn.BCEWithLogitsLoss() \n        self.validation_step_outputs = []\n        self.lin = nn.Softmax(dim=1)\n        self.best_score = 1000.0\n    def forward(self,eeg, spec):\n        logits = self.backbone(eeg, spec)\n        # logits = self.lin(logits)\n        return logits\n        \n    def configure_optimizers(self):\n        return get_optimizer(lr=Config.LR, params=self.parameters())\n\n    def training_step(self, batch, batch_idx):\n        eeg, spec, target = batch        \n\n        y_pred = self(eeg, spec)\n        loss = self.loss_function(y_pred,target)\n\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, prog_bar=True)\n        return loss        \n\n    def validation_step(self, batch, batch_idx):\n        eeg, spec, target = batch \n        # print(target)\n        y_pred = self(eeg, spec)\n        val_loss = self.loss_function(y_pred, target)\n        self.log(\"val_loss\", val_loss, on_step=True, on_epoch=True, logger=True, prog_bar=True)\n        self.validation_step_outputs.append({\"val_loss\": val_loss, \"logits\": y_pred, \"targets\": target})\n\n        return {\"val_loss\": val_loss, \"logits\": y_pred, \"targets\": target}\n    \n    def train_dataloader(self):\n        return self._train_dataloader \n    \n    def validation_dataloader(self):\n        return self._validation_dataloader\n    \n    def on_validation_epoch_end(self):\n        outputs = self.validation_step_outputs\n        # print(len(outputs))\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        output_val = nn.Softmax(dim=1)(torch.cat([x['logits'] for x in outputs],dim=0)).cpu().detach().numpy()\n        target_val = torch.cat([x['targets'] for x in outputs],dim=0).cpu().detach().numpy()\n        self.validation_step_outputs = []\n\n        val_df = pd.DataFrame(target_val, columns = list(TARGETS))\n        pred_df = pd.DataFrame(output_val, columns = list(TARGETS))\n\n        val_df['id'] = [f'id_{i}' for i in range(len(val_df))] \n        pred_df['id'] = [f'id_{i}' for i in range(len(pred_df))] \n\n        avg_score = avg_loss\n        # avg_score = score(val_df, pred_df, row_id_column_name = 'id')\n\n        if avg_score < self.best_score:\n            print(f'Fold {self.fold}: Epoch {self.current_epoch} validation loss {avg_loss}')\n            print(f'Fold {self.fold}: Epoch {self.current_epoch} validation KDL score {avg_score}')\n            self.best_score = avg_score\n            # val_df.to_csv(f'{Config.output_dir}/val_df_f{self.fold}.csv',index=False)\n            # pred_df.to_csv(f'{Config.output_dir}/pred_df_f{self.fold}.csv',index=False)\n        \n        return {'val_loss': avg_loss,'val_cmap':avg_score}\n    \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(data_loader, model):\n        \n    model.to('cuda')\n    model.eval()    \n    predictions = []\n    for batch in tqdm(data_loader):\n\n        with torch.no_grad():\n            x, spec = batch\n            x = x.cuda()\n            spec = spec.cuda()\n            \n            # inputs = {key:val.reshape(val.shape[0], -1).to(config.device) for key,val in batch.items()}\n            outputs,_,_,out1d,out2d = model(x, spec)\n            outputs = outputs*0.5 + out1d*0.25 + out2d*0.25\n            outputs = nn.Softmax(dim=1)(outputs)\n        predictions.extend(outputs.detach().cpu().numpy())\n    predictions = np.vstack(predictions)\n    return predictions\n\ndef predict_nobatch(ds_test, model):\n    \n    model.to('cuda')\n    model.eval()    \n    predictions = []\n    for en in tqdm(range(len(ds_test))):\n        # print(en)\n        x = ds_test[en]\n        x = x.unsqueeze(0).cuda()\n        # print(images.shape)\n        with torch.no_grad():\n            outputs = model(x)\n            outputs = nn.Softmax(dim=1)(outputs)\n            outputs = outputs.detach().cpu().numpy()\n\n        predictions.append(outputs)\n        \n    return predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.loggers import WandbLogger\nimport gc\ntorch.set_float32_matmul_precision('high')\ndef run_inference(fold_id, Config):\n    print(f\"Running training for fold {fold_id}...\")\n    logger = None\n    pred_cols = [f'pred_{t}' for t in TARGETS]\n    df_test = test.copy()\n    dl_test, ds_test = get_test_dls(df_test)\n    \n\n    print(f\"Running inference model '{Config.model_dir}/eegnet_best_loss_fold{fold_id}.ckpt'..\")\n    \n    model = EEGModel.load_from_checkpoint(f'{Config.model_dir}/eegnet_best_loss_fold{fold_id}.ckpt',map_location='cuda:0',\n                                          train_dataloader=None,validation_dataloader=None,config=Config)\n    \n    preds = predict(dl_test, model)  \n    print(preds.shape)\n    gc.collect()\n    # torch.cuda.empty_cache()\n    return preds\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\nfor en,f in enumerate(Config.trn_folds):\n    preds = run_inference(f, Config)\n    test_preds.append(preds)\ntest_preds = np.mean(test_preds, 0)\ntest_preds.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE SUBMISSION.CSV\nfrom IPython.display import display\n\nsub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = pred_origin*0.5 + pred_c*0.2 + test_preds*0.3\nsub.to_csv('submission.csv',index=False)\nprint('Submission shape',sub.shape)\ndisplay( sub.head() )","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:45:04.28164Z","iopub.execute_input":"2024-03-30T05:45:04.282013Z","iopub.status.idle":"2024-03-30T05:45:04.302704Z","shell.execute_reply.started":"2024-03-30T05:45:04.281979Z","shell.execute_reply":"2024-03-30T05:45:04.301561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nprint(sub.iloc[:,-6:].sum(axis=1))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T05:45:04.304116Z","iopub.execute_input":"2024-03-30T05:45:04.304417Z","iopub.status.idle":"2024-03-30T05:45:04.311319Z","shell.execute_reply.started":"2024-03-30T05:45:04.304392Z","shell.execute_reply":"2024-03-30T05:45:04.310427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}