{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7585255,"sourceType":"datasetVersion","datasetId":4415285}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2024-04-05T11:58:26.492291Z","iopub.execute_input":"2024-04-05T11:58:26.493162Z","iopub.status.idle":"2024-04-05T11:58:30.070529Z","shell.execute_reply.started":"2024-04-05T11:58:26.493128Z","shell.execute_reply":"2024-04-05T11:58:30.069557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntrain = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n\nprint(\"Column names:\")\nprint(train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T11:58:56.162861Z","iopub.execute_input":"2024-04-05T11:58:56.163296Z","iopub.status.idle":"2024-04-05T11:58:56.808417Z","shell.execute_reply.started":"2024-04-05T11:58:56.163270Z","shell.execute_reply":"2024-04-05T11:58:56.807481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Step 1: Data Preparation\nclass EEGDataset(Dataset):\n    def __init__(self, csv_file, transform=None):\n        self.data = pd.read_csv(csv_file)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        # preprocess EEG data and labels\n        eeg_data = self.data.iloc[idx, 0:10]  # first 10 columns are EEG data\n        label = self.data.iloc[idx, -1]  # Assuming the last column is the label\n        if self.transform:\n            eeg_data = self.transform(eeg_data)\n        return eeg_data, label\n\nclass EEGClassifier(nn.Module):\n    def __init__(self, input_size, hidden_size1, hidden_size2, num_classes):\n        super(EEGClassifier, self).__init__()\n        self.fc1 = nn.Linear(input_size, hidden_size1)\n        self.fc2 = nn.Linear(hidden_size1, hidden_size2)\n        self.fc3 = nn.Linear(hidden_size2, num_classes)\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n    \n# Step 3: Training\ndef train_model(model, train_loader, criterion, optimizer, num_epochs):\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        for inputs, labels in train_loader:\n            inputs = inputs.float()  # Convert inputs to float tensor\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item() * inputs.size(0)\n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f'Epoch {epoch+1}/{num_epochs}, Loss: {epoch_loss:.4f}')\n        \n        \ndef evaluate_model(model, test_loader):\n    model.eval()\n    all_predictions = []\n    all_labels = []\n    with torch.no_grad():\n        for inputs, labels in test_loader:\n            inputs = inputs.float()  # Convert inputs to float tensor\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            all_predictions.extend(predicted.tolist())\n            all_labels.extend(labels.tolist())\n\n    accuracy = accuracy_score(all_labels, all_predictions)\n    precision = precision_score(all_labels, all_predictions, average='weighted')\n    recall = recall_score(all_labels, all_predictions, average='weighted')\n    f1 = f1_score(all_labels, all_predictions, average='weighted')\n\n    print(f'Accuracy: {accuracy:.4f}, Precision: {precision:.4f}, Recall: {recall:.4f}, F1-score: {f1:.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-05T11:58:57.949572Z","iopub.execute_input":"2024-04-05T11:58:57.949939Z","iopub.status.idle":"2024-04-05T11:58:58.452031Z","shell.execute_reply.started":"2024-04-05T11:58:57.949910Z","shell.execute_reply":"2024-04-05T11:58:58.451007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_size = 10  \nhidden_size1 = 64  \nhidden_size2 = 32  \nnum_classes = 2 \n\n\ndataset = EEGDataset('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ntrain_loader = DataLoader(dataset, batch_size=64, shuffle=True)\n\nmodel = EEGClassifier(input_size, hidden_size1, hidden_size2, num_classes)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:12:47.782643Z","iopub.execute_input":"2024-04-05T12:12:47.783042Z","iopub.status.idle":"2024-04-05T12:12:48.115873Z","shell.execute_reply.started":"2024-04-05T12:12:47.783009Z","shell.execute_reply":"2024-04-05T12:12:48.115001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is based on the [EfficientNetB0 Starter](https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43) notebook, modified for Pytorch. The original notebook is implemented in Tensorflow.\n\n* Change to pytorch's Dataset and Dataloader\n* Use efficientnet_b0 from torchvision\n* Use pytorch lightning for building the model and training\n* Inference using Trainer on multiple GPUs (DDP strategy) requires adding predictions gathering code, otherwise it will hang waiting for other nodes. So the raw pytorch's inference loop is used in the CV part. This needs to be done after training all the folds since the manual torch's GPU device initialization can't be mixed with lightning's DDP strategy.","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport gc\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models import efficientnet_b0, EfficientNet_B0_Weights\nimport pytorch_lightning as pl\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport albumentations as albu\nfrom sklearn.model_selection import KFold, GroupKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-05T12:12:52.022340Z","iopub.execute_input":"2024-04-05T12:12:52.023151Z","iopub.status.idle":"2024-04-05T12:12:52.029018Z","shell.execute_reply.started":"2024-04-05T12:12:52.023120Z","shell.execute_reply":"2024-04-05T12:12:52.028045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"VER = 5\n# IF THIS EQUALS NONE, THEN WE TRAIN NEW MODELS\n# IF THIS EQUALS DISK PATH, THEN WE LOAD PREVIOUSLY TRAINED MODELS\nLOAD_MODELS_FROM = '/kaggle/input/hms-efficientnetb0-pt-ckpts/'\n\nUSE_KAGGLE_SPECTROGRAMS = True\nUSE_EEG_SPECTROGRAMS = True","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-05T12:12:54.182412Z","iopub.execute_input":"2024-04-05T12:12:54.183102Z","iopub.status.idle":"2024-04-05T12:12:54.187303Z","shell.execute_reply.started":"2024-04-05T12:12:54.183071Z","shell.execute_reply":"2024-04-05T12:12:54.186361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint('Train shape:', df.shape )\nprint('Targets', list(TARGETS))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:12:54.522442Z","iopub.execute_input":"2024-04-05T12:12:54.522800Z","iopub.status.idle":"2024-04-05T12:12:54.727370Z","shell.execute_reply.started":"2024-04-05T12:12:54.522771Z","shell.execute_reply":"2024-04-05T12:12:54.726497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df.groupby('eeg_id')[\n    ['spectrogram_id', 'spectrogram_label_offset_seconds']\n].agg({'spectrogram_id': 'first', 'spectrogram_label_offset_seconds': 'min'})\ntrain.columns = ['spec_id', 'min']\n\ntmp = df.groupby('eeg_id')[\n    ['spectrogram_id','spectrogram_label_offset_seconds']\n].agg({'spectrogram_label_offset_seconds' :'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1, keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:12:57.172659Z","iopub.execute_input":"2024-04-05T12:12:57.173007Z","iopub.status.idle":"2024-04-05T12:12:57.256241Z","shell.execute_reply.started":"2024-04-05T12:12:57.172971Z","shell.execute_reply":"2024-04-05T12:12:57.255370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"READ_SPEC_FILES = False\n\n# READ ALL SPECTROGRAMS\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/'\nfiles = os.listdir(PATH)\nprint(f'There are {len(files)} spectrogram parquets')\n\nif READ_SPEC_FILES:    \n    spectrograms = {}\n    for i,f in enumerate(files):\n        if i % 100 == 0:\n            print(i, ', ', end='')\n        tmp = pd.read_parquet(f'{PATH}{f}')\n        name = int(f.split('.')[0])\n        spectrograms[name] = tmp.iloc[:,1:].values\nelse:\n    spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:12:58.987642Z","iopub.execute_input":"2024-04-05T12:12:58.988020Z","iopub.status.idle":"2024-04-05T12:14:01.152571Z","shell.execute_reply.started":"2024-04-05T12:12:58.987971Z","shell.execute_reply":"2024-04-05T12:14:01.151729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"READ_EEG_SPEC_FILES = False\n\nif READ_EEG_SPEC_FILES:\n    all_eegs = {}\n    for i,e in enumerate(train.eeg_id.values):\n        if i % 100 == 0:\n            print(i, ', ', end='')\n        x = np.load(f'/kaggle/input/brain-eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n        all_eegs[e] = x\nelse:\n    all_eegs = np.load('/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:14:39.997705Z","iopub.execute_input":"2024-04-05T12:14:39.998064Z","iopub.status.idle":"2024-04-05T12:16:02.322273Z","shell.execute_reply.started":"2024-04-05T12:14:39.998037Z","shell.execute_reply":"2024-04-05T12:16:02.316667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- __getitem__ calling another method __getitems__ is a design choice . The __getitem__ method is a special method in Python, and when you use it, it's customary to handle the logic directly within that method.\nBy having __getitem__ delegate the actual work to __getitems__, it allows for better code organization and potentially easier maintenance or extension in the future.  Using such additional methods can add clarity to the code by separating different concerns or responsibilities.\n\n- def __getitem__(self, index):: define  method named __getitem__ which takes two parameters: self (a reference to the instance of the class) and index (the index used to access the item).\n- return self.__getitems__([index]): Inside the method, it calls another method __getitems__ of the same object (instance of the class) with a list containing the index as its only element. Then, it returns the result of this call.\n\n- whenever an instance of this class is accessed using square brackets, like instance[index], it will internally call __getitems__ method with the given index and return the result. The actual functionality of accessing items is delegated to __getitems__\n\n``` def __getitems__(self, indices):\n        X, y = self._generate_data(indices)\n        if self.augment:\n            X = self._augment(X) \n        if self.mode == 'train':\n            return list(zip(X, y))\n        else:\n            return X```\n","metadata":{}},{"cell_type":"markdown","source":"### generate_data()\n   - X is a 4D array with dimensions (len(indexes), 128, 256, 8). It will store data related to the spectrogram images.\n   - y is a 2D array with dimensions (len(indexes), 6). It will store labels or target values.\n   - img is a 2D array with dimensions (128, 256). It is initialized with all ones, presumably to be used for storing temporary data.\n   \n   \n   - iterate over the indexes using enumerate(), obtaining both the index j and the value i. Then it fetches the row from data source (self.data) using index i. If the mode is 'test', r is set to 0; otherwise, it calculates r based on logic involving the values of 'min' and 'max' from the row.\n   \n   \n- for k in range(4) - \n    - loop iterates over k from 0 to 3 (inclusive).\n    - It extracts a portion of the spectrogram image stored in self.specs based on the value of row.spec_id.\n    - r indicates the starting row index, and r+300 indicates the ending row index of the extracted portion.\n    - k*100 and (k+1)*100 indicate the starting and ending column indices, respectively.\n    - .T transposes the extracted portion of the image.\n    \n- LOG TRANSFORM SPECTROGRAM\n    - This part of code  clips the values of img between np.exp(-4) and np.exp(8).\n    - It then applies a logarithmic transformation to img.\n    \n- STANDARDIZE PER IMAGE\n    - This part of the code calculates the mean m and standard deviation s of the flattened img, ignoring any NaN values.\n    - It standardizes img by subtracting the mean and dividing by the standard deviation.\n    - np.nan_to_num() replaces any NaN values in img with 0.0.\n    \n- CROP TO 256 TIME STEPS\n    - It assigns the standardized and cropped img to a slice of the array X.\nThe slicing operation [14:-14] crops the rows from 14th to 14th from the end, and [:, 22:-22] crops the columns from 22nd to 22nd from the end.\n    - / 2.0 normalizes the values in the cropped image.\n    \n- EEG SPECTROGRAMS\n    - It fetches an EEG spectrogram from self.eeg_specs based on the value of row.eeg_id.\n    - It assigns the EEG spectrogram to a slice of X in the last four channels.\n    \n```if self.mode != 'test':\n    y[j,] = row[TARGETS] ``` - If the mode is not 'test', it assigns the target values from row to y.","metadata":{}},{"cell_type":"code","source":"TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS2 = {x: y for y, x in TARS.items()}\n\n\nclass EEGDataset(Dataset):\n    \n    def __init__(self, data, augment=False, mode='train', specs=spectrograms, eeg_specs=all_eegs): \n        self.data = data\n        self.augment = augment\n        self.mode = mode\n        self.specs = specs\n        self.eeg_specs = eeg_specs\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        return self.__getitems__([index])\n    \n    def __getitems__(self, indices):\n        X, y = self._generate_data(indices)\n        if self.augment:\n            X = self._augment(X) \n        if self.mode == 'train':\n            return list(zip(X, y))\n        else:\n            return X\n    \n    def _generate_data(self, indexes):\n        X = np.zeros((len(indexes), 128, 256, 8),dtype='float32')\n        y = np.zeros((len(indexes), 6),dtype='float32')\n        img = np.ones((128, 256),dtype='float32')\n        \n        for j, i in enumerate(indexes):\n            row = self.data.iloc[i]\n            if self.mode == 'test': \n                r = 0\n            else: \n                r = int((row['min'] + row['max'])//4)\n\n            for k in range(4):\n                # EXTRACT 300 ROWS OF SPECTROGRAM\n                img = self.specs[row.spec_id][r:r+300, k*100:(k+1)*100].T\n                \n                # LOG TRANSFORM SPECTROGRAM\n                img = np.clip(img, np.exp(-4), np.exp(8))\n                img = np.log(img)\n                \n                # STANDARDIZE PER IMAGE\n                ep = 1e-6\n                m = np.nanmean(img.flatten())\n                s = np.nanstd(img.flatten())\n                img = (img - m) / (s + ep)\n                img = np.nan_to_num(img, nan=0.0)\n                \n                # CROP TO 256 TIME STEPS\n                X[j, 14:-14, :, k] = img[:, 22:-22] / 2.0\n        \n            # EEG SPECTROGRAMS\n            img = self.eeg_specs[row.eeg_id]\n            X[j, :, :, 4:] = img\n                \n            if self.mode != 'test':\n                y[j,] = row[TARGETS]\n            \n        return X, y\n    \n    def _random_transform(self, img):\n        composition = albu.Compose([\n            albu.HorizontalFlip(p=0.5),\n            # albu.CoarseDropout(max_holes=8,max_height=32,max_width=32,fill_value=0,p=0.5),\n        ])\n        return composition(image=img)['image']\n            \n    def __augment(self, img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i,] = self._random_transform(img_batch[i,])\n        return img_batch","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:21:48.964537Z","iopub.execute_input":"2024-04-05T12:21:48.964901Z","iopub.status.idle":"2024-04-05T12:21:48.981936Z","shell.execute_reply.started":"2024-04-05T12:21:48.964872Z","shell.execute_reply":"2024-04-05T12:21:48.981050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = EEGDataset(train)\ndataloader = DataLoader(dataset, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:21:49.212193Z","iopub.execute_input":"2024-04-05T12:21:49.212500Z","iopub.status.idle":"2024-04-05T12:21:49.216847Z","shell.execute_reply.started":"2024-04-05T12:21:49.212476Z","shell.execute_reply":"2024-04-05T12:21:49.215967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### visualize data from PyTorch DataLoader\n- for i, (x, y) in enumerate(dataloader):: This iterates over the data loader yielding batches of input data x and corresponding labels y. enumerate adds a counter to the iterator, so i will be the index of the current batch.\n - plt.figure(figsize=(20, 8)): This initializes a new figure for plotting with a specified figure size.\n - for j in range(ROWS):: This iterates over the rows of the subplot grid.\n - for k in range(COLS):: This iterates over the columns of the subplot grid.\n - plt.subplot(ROWS, COLS, j*COLS + k + 1): This specifies the subplot to plot the image in. It calculates the index of the subplot based on the current row (j), column (k), and the total number of columns (COLS).\n - t = y[j*COLS + k]: This retrieves the target label for the current subplot.\n - img = torch.flip(x[j*COLS+k, :, :, 0], (0,)): This flips the image tensor along the 0th dimension. It seems to be processing the input image data.\n - mn = img.flatten().min() and mx = img.flatten().max(): These lines find the minimum and maximum values in the flattened image tensor.\n - img = (img-mn)/(mx-mn): This normalizes the image tensor between 0 and 1.\n - for s in t[1:]: tars += f', {s:0.2f}': This prepares a string representation of the target label.\n - eeg = train.eeg_id.values[i*32+j*COLS+k]: This line seems to fetch some EEG-related data based on the current index i, j, and k. It's assuming there's a dataframe named train with a column named eeg_id.\n - plt.show(): This displays the entire figure with all subplots.\n - if i == BATCHES-1: break: break the loop after processing a certain number of batches (BATCHES).","metadata":{}},{"cell_type":"code","source":"ROWS = 2\nCOLS = 3\nBATCHES = 2\n\nfor i, (x, y) in enumerate(dataloader):\n    plt.figure(figsize=(20, 8))\n    for j in range(ROWS):\n        for k in range(COLS):\n            plt.subplot(ROWS, COLS, j*COLS + k + 1)\n            t = y[j*COLS + k]\n            img = torch.flip(x[j*COLS+k, :, :, 0], (0,))\n            mn = img.flatten().min()\n            mx = img.flatten().max()\n            img = (img-mn)/(mx-mn)\n            plt.imshow(img)\n            tars = f'[{t[0]:0.2f}]'\n            for s in t[1:]:\n                tars += f', {s:0.2f}'\n            eeg = train.eeg_id.values[i*32+j*COLS+k]\n            plt.title(f'EEG = {eeg}\\nTarget = {tars}',size=12)\n            plt.yticks([])\n            plt.ylabel('Frequencies (Hz)',size=14)\n            plt.xlabel('Time (sec)',size=16)\n    plt.show()\n    if i == BATCHES-1:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:21:52.382623Z","iopub.execute_input":"2024-04-05T12:21:52.383001Z","iopub.status.idle":"2024-04-05T12:21:54.801204Z","shell.execute_reply.started":"2024-04-05T12:21:52.382957Z","shell.execute_reply":"2024-04-05T12:21:54.800024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dataset, dataloader\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:26:09.192966Z","iopub.execute_input":"2024-04-05T12:26:09.193360Z","iopub.status.idle":"2024-04-05T12:26:09.405716Z","shell.execute_reply.started":"2024-04-05T12:26:09.193331Z","shell.execute_reply":"2024-04-05T12:26:09.404741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training\n\n- __init__ method:\n     - initializes the neural network model.\n     - calls the __init__ method of the superclass.\n     - It initializes self.base_model with an instance of the EfficientNet B0 model (presumably from some library or module).\n     - It loads pre-trained weights for the EfficientNet B0 model from a file specified by WEIGHTS_FILE.\n     - It modifies the last fully connected layer of the base model to output 6 classes. The number of input features to this layer is inferred from the existing layer's configuration.\n     - It initializes self.prob_out with an instance of Softmax activation function.\n    \n### forward method:\n    - It splits the input x into two parts (x1 and x2) along the last dimension (x is a 4-dimensional tensor).\n    - It concatenates x1 and x2 along the second dimension.\n    - Depending on the values of USE_KAGGLE_SPECTROGRAMS and USE_EEG_SPECTROGRAMS, it selects the appropriate part (x1, x2, or both concatenated) as the input x for further processing.\n    - It duplicates the selected input (x) along the channel dimension (dim=3).\n    - It permutes the dimensions of x to bring the channel dimension to the second position.\n    - It passes the permuted x through the base_model and returns the output.\n    \n### training_step method:\n    - single training step.\n    - It takes a batch of input samples and their corresponding labels (x and y).\n    - It computes the output of the model (out) by calling the forward method.\n    - It applies the log softmax function along dimension 1 to the output.\n    - It defines a Kullback-Leibler divergence loss (kl_loss) with reduction set to 'batchmean'.\n    - It computes the loss by applying the KL divergence loss between the log softmax output and the ground truth labels (y).\n    - return the computed loss.\n    \n### predict_step method:\n    - generate predictions during inference.\n    - It takes a batch of input samples and their corresponding labels, along with batch_idx and dataloader_idx.\n    - It computes the softmax of the model's output along dimension 1 and returns the result.\n\n### configure_optimizers method:\n    - configuring the optimizer(s) to be used during training.\n    - It initializes an Adam optimizer with a learning rate of 1e-3, optimizing the parameters of the model.\n    - It returns the optimizer.\n    \n ### Normalization of Output:\n     - Softmax function is commonly used to normalize the output of a model into a probability distribution over multiple classes.\n     - By applying softmax, each element in the output vector is transformed into a value between 0 and 1, representing the probability of the corresponding class.","metadata":{}},{"cell_type":"code","source":"# Original Code\n# WEIGHTS_FILE = '/kaggle/input/hms-efficientnetb0-pt-ckpts/efficientnet_b0_rwightman-7f5810bc.pth'\n\n\n# class EEGEffnetB0(pl.LightningModule):\n    \n#     def __init__(self):\n#         super().__init__()\n#         self.base_model = efficientnet_b0()\n#         self.base_model.load_state_dict(torch.load(WEIGHTS_FILE))\n#         self.base_model.classifier[1] = nn.Linear(self.base_model.classifier[1].in_features, 6, dtype=torch.float32)\n#         self.prob_out = nn.Softmax()\n        \n#     def forward(self, x):\n#         x1 = [x[:, :, :, i:i+1] for i in range(4)]\n#         x1 = torch.concat(x1, dim=1)\n#         x2 = [x[:, :, :, i+4:i+5] for i in range(4)]\n#         x2 = torch.concat(x2, dim=1)\n        \n#         if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n#             x = torch.concat([x1, x2], dim=2)\n#         elif USE_EEG_SPECTROGRAMS:\n#             x = x2\n#         else:\n#             x = x1\n#         x = torch.concat([x, x, x], dim=3)\n#         x = x.permute(0, 3, 1, 2)\n        \n#         out = self.base_model(x)\n#         return out\n    \n#     def training_step(self, batch, batch_idx):\n#         x, y = batch\n#         out = self.forward(x)\n#         out = F.log_softmax(out, dim=1)\n#         kl_loss = nn.KLDivLoss(reduction='batchmean')\n#         loss = kl_loss(out, y)\n#         return loss\n    \n#     def predict_step(self, batch, batch_idx, dataloader_idx=0):\n#         return F.softmax(self(batch), dim=1)\n    \n#     def configure_optimizers(self):\n#         optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)\n#         return optimizer","metadata":{"execution":{"iopub.status.busy":"2024-04-05T05:19:04.137283Z","iopub.execute_input":"2024-04-05T05:19:04.137642Z","iopub.status.idle":"2024-04-05T05:19:04.149598Z","shell.execute_reply.started":"2024-04-05T05:19:04.137615Z","shell.execute_reply":"2024-04-05T05:19:04.148668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Preload EfficientNet weights\nWEIGHTS_FILE = '/kaggle/input/hms-efficientnetb0-pt-ckpts/efficientnet_b0_rwightman-7f5810bc.pth'\nbase_model = efficientnet_b0()\nbase_model.load_state_dict(torch.load(WEIGHTS_FILE))\n\nclass EEGEffnetB0(pl.LightningModule):\n    \n    def __init__(self):\n        super().__init__()\n        self.base_model = base_model\n        self.base_model.classifier[1] = nn.Linear(self.base_model.classifier[1].in_features, 6, dtype=torch.float32)\n        self.prob_out = nn.Softmax()\n        \n    def forward(self, x):\n        x1 = [x[:, :, :, i:i+1] for i in range(4)]\n        x1 = torch.cat(x1, dim=1)\n        x2 = [x[:, :, :, i+4:i+5] for i in range(4)]\n        x2 = torch.cat(x2, dim=1)\n        \n        if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n            x = torch.cat([x1, x2], dim=2)\n        elif USE_EEG_SPECTROGRAMS:\n            x = x2\n        else:\n            x = x1\n        x = torch.cat([x, x, x], dim=3)\n        x = x.permute(0, 3, 1, 2)\n        \n        out = self.base_model(x)\n        return out\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        out = self.forward(x)\n        out = F.log_softmax(out, dim=1)\n        kl_loss = nn.KLDivLoss(reduction='batchmean')\n        loss = kl_loss(out, y)\n        self.log('train_loss', loss)\n        return loss\n    \n    def predict_step(self, batch, batch_idx, dataloader_idx=0):\n        return F.softmax(self(batch), dim=1)\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, verbose=True)\n        return {'optimizer': optimizer, 'lr_scheduler': scheduler, 'monitor': 'train_loss'}\n","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:26:58.142516Z","iopub.execute_input":"2024-04-05T12:26:58.143134Z","iopub.status.idle":"2024-04-05T12:26:58.589427Z","shell.execute_reply.started":"2024-04-05T12:26:58.143100Z","shell.execute_reply":"2024-04-05T12:26:58.588551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch.nn.functional as F\n# from torch.utils.data import DataLoader\n\n# # Assuming USE_KAGGLE_SPECTROGRAMS, USE_EEG_SPECTROGRAMS are defined\n\n# class EEGEffnetB0(pl.LightningModule):\n    \n#     def __init__(self, weights_file):\n#         super().__init__()\n#         self.base_model = efficientnet_b0()\n#         self.base_model.load_state_dict(torch.load(weights_file))\n#         # Assuming efficientnet_b0 is imported and defined elsewhere\n#         self.base_model.classifier[1] = nn.Linear(self.base_model.classifier[1].in_features, 6, dtype=torch.float32)\n#         self.prob_out = nn.Softmax()\n        \n#     def forward(self, x):\n#         x1 = x[:, :, :, :4]\n#         x2 = x[:, :, :, 4:]\n        \n#         if USE_KAGGLE_SPECTROGRAMS & USE_EEG_SPECTROGRAMS:\n#             x = torch.cat([x1, x2], dim=2)\n#         elif USE_EEG_SPECTROGRAMS:\n#             x = x2\n#         else:\n#             x = x1\n        \n#         x = torch.cat([x, x, x], dim=3)\n#         x = x.permute(0, 3, 1, 2)\n        \n#         out = self.base_model(x)\n#         return out\n    \n#     def training_step(self, batch, batch_idx):\n#         x, y = batch\n#         out = self.forward(x)\n#         out = F.log_softmax(out, dim=1)\n#         kl_loss = nn.KLDivLoss(reduction='batchmean')\n#         loss = kl_loss(out, y)\n#         return loss\n    \n#     def predict_step(self, batch, batch_idx, dataloader_idx=0):\n#         return F.softmax(self(batch), dim=1)\n    \n#     def configure_optimizers(self):\n#         optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)\n#         return optimizer\n\n# # Usage:\n# WEIGHTS_FILE = '/kaggle/input/hms-efficientnetb0-pt-ckpts/efficientnet_b0_rwightman-7f5810bc.pth'\n# model = EEGEffnetB0(weights_file=WEIGHTS_FILE)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-05T05:34:24.413601Z","iopub.execute_input":"2024-04-05T05:34:24.414351Z","iopub.status.idle":"2024-04-05T05:34:24.611141Z","shell.execute_reply.started":"2024-04-05T05:34:24.414320Z","shell.execute_reply":"2024-04-05T05:34:24.610081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Original Code\nall_oof = []\nall_true = []\nvalid_loaders = []\n\ngkf = GroupKFold(n_splits=5)\nfor i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n    print('#'*25)\n    print(f'### Fold {i+1}')\n    \n    train_ds = EEGDataset(train.iloc[train_index])\n    train_loader = DataLoader(train_ds, shuffle=True, batch_size=32, num_workers=3)\n    valid_ds = EEGDataset(train.iloc[valid_index], mode='valid')\n    valid_loader = DataLoader(valid_ds, shuffle=False, batch_size=64, num_workers=3)\n    \n    print(f'### Train size: {len(train_index)}, Valid size: {len(valid_index)}')\n    print('#'*25)\n    \n    trainer = pl.Trainer(max_epochs=4)\n    model = EEGEffnetB0()\n    if LOAD_MODELS_FROM is None:\n        trainer.fit(model=model, train_dataloaders=train_loader)\n        trainer.save_checkpoint(f'EffNet_v{VER}_f{i}.ckpt')\n\n    valid_loaders.append(valid_loader)\n    all_true.append(train.iloc[valid_index][TARGETS].values)\n    del trainer, model\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:27:02.392543Z","iopub.execute_input":"2024-04-05T12:27:02.392909Z","iopub.status.idle":"2024-04-05T12:27:04.753141Z","shell.execute_reply.started":"2024-04-05T12:27:02.392879Z","shell.execute_reply":"2024-04-05T12:27:04.752208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_oof = []\n# all_true = []\n# valid_loaders = []\n\n# gkf = GroupKFold(n_splits=5)\n# trainer = pl.Trainer(max_epochs=4)\n\n# for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):  \n#     train_ds = EEGDataset(train.iloc[train_index])\n#     train_loader = DataLoader(train_ds, shuffle=True, batch_size=32, num_workers=3)\n#     valid_ds = EEGDataset(train.iloc[valid_index], mode='valid')\n#     valid_loader = DataLoader(valid_ds, shuffle=False, batch_size=32, num_workers=3)  # Ensure consistent batch size\n    \n#     print(f'### Fold {i+1}')\n\n#     if LOAD_MODELS_FROM is None:\n#         model = EEGEffnetB0()\n#         trainer.fit(model=model, train_dataloaders=train_loader)\n#         trainer.save_checkpoint(f'EffNet_v{VER}_f{i}.ckpt')\n    \n#     valid_loaders.append(valid_loader)\n#     all_true.append(train.iloc[valid_index][TARGETS].values)\n\n# # Clean up\n# del trainer, model\n# gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-05T05:38:25.573635Z","iopub.execute_input":"2024-04-05T05:38:25.573997Z","iopub.status.idle":"2024-04-05T05:38:25.578922Z","shell.execute_reply.started":"2024-04-05T05:38:25.573970Z","shell.execute_reply":"2024-04-05T05:38:25.577921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:37:04.342515Z","iopub.execute_input":"2024-04-05T12:37:04.343295Z","iopub.status.idle":"2024-04-05T12:37:04.347498Z","shell.execute_reply.started":"2024-04-05T12:37:04.343260Z","shell.execute_reply":"2024-04-05T12:37:04.346495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### perform validation on each fold of the cross-validated model and collects predictions and true labels for further analysis.\n\n- Load Model Checkpoint:\n      - It constructs the filename of the model checkpoint (ckpt_file) based on the fold number (i) and the version (VER) of the model.\n      - If LOAD_MODELS_FROM is None, it assumes that the checkpoint is located in the current directory. Otherwise, it assumes a specific directory.\n      - It loads the model from the checkpoint using EEGEffnetB0.load_from_checkpoint(). This assumes that EEGEffnetB0 is a PyTorch Lightning module.\n      - It moves the model to the specified device (likely GPU) and sets the model to evaluation mode.\n      \n- Validation Loop:\n       - Within the context of torch.inference_mode(), it iterates over the validation data loader corresponding to the current fold (valid_loaders[i]).\n       - For each batch in the validation data loader, it moves the batch to the specified device (val_batch.to(device)).\n       - It passes the batch through the model (model(val_batch)), applies softmax along dimension 1, moves the result to CPU (cpu()), and converts it to a NumPy array (numpy()).\n       - It appends the resulting predictions (oof) to the list all_oof.\n       \n- Cleanup:\n       - After validating all batches for the current fold, delete the model to free up memory and runs garbage collection.\n- Concatenate Predictions and True Labels:\n       - After validating all folds, it concatenates all predictions (all_oof) and true labels (all_true) along the appropriate axis to prepare for further analysis.","metadata":{}},{"cell_type":"code","source":"for i in range(5):\n    print('#'*25)\n    print(f'### Validating Fold {i+1}')\n\n    ckpt_file = f'EffNet_v{VER}_f{i}.ckpt' if LOAD_MODELS_FROM is None else f'{LOAD_MODELS_FROM}/EffNet_v{VER}_f{i}.ckpt'\n    model = EEGEffnetB0.load_from_checkpoint(ckpt_file)\n    model.to(device).eval()\n    with torch.inference_mode():\n        for val_batch in valid_loaders[i]:\n            val_batch = val_batch.to(device)\n            oof = torch.softmax(model(val_batch), dim=1).cpu().numpy()\n            all_oof.append(oof)\n    del model\n    gc.collect()\n\nall_oof = np.concatenate(all_oof)\nall_true = np.concatenate(all_true)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:37:06.502454Z","iopub.execute_input":"2024-04-05T12:37:06.502805Z","iopub.status.idle":"2024-04-05T12:38:29.470558Z","shell.execute_reply.started":"2024-04-05T12:37:06.502778Z","shell.execute_reply":"2024-04-05T12:38:29.469442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = pd.DataFrame(all_oof.copy())\noof['id'] = np.arange(len(oof))\n\ntrue = pd.DataFrame(all_true.copy())\ntrue['id'] = np.arange(len(true))\n\ncv = score(solution=true, submission=oof, row_id_column_name='id')\nprint('CV Score KL-Div for EfficientNetB2 =',cv)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T05:44:01.173589Z","iopub.execute_input":"2024-04-05T05:44:01.174353Z","iopub.status.idle":"2024-04-05T05:44:01.234829Z","shell.execute_reply.started":"2024-04-05T05:44:01.174318Z","shell.execute_reply":"2024-04-05T05:44:01.233940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# del all_eegs, spectrograms\ngc.collect()\n\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:38:36.864236Z","iopub.execute_input":"2024-04-05T12:38:36.865196Z","iopub.status.idle":"2024-04-05T12:38:37.074592Z","shell.execute_reply.started":"2024-04-05T12:38:36.865157Z","shell.execute_reply":"2024-04-05T12:38:37.073647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\nfiles2 = os.listdir(PATH2)\nprint(f'There are {len(files2)} test spectrogram parquets')\n    \nspectrograms2 = {}\nfor i, f in enumerate(files2):\n    if i % 100 == 0:\n        print(i, ', ',end='')\n    tmp = pd.read_parquet(f'{PATH2}{f}')\n    name = int(f.split('.')[0])\n    spectrograms2[name] = tmp.iloc[:, 1:].values\n    \n# RENAME FOR DATALOADER\ntest = test.rename({'spectrogram_id': 'spec_id'}, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:38:43.334351Z","iopub.execute_input":"2024-04-05T12:38:43.335041Z","iopub.status.idle":"2024-04-05T12:38:43.660948Z","shell.execute_reply.started":"2024-04-05T12:38:43.335007Z","shell.execute_reply":"2024-04-05T12:38:43.660160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n- maddest(d, axis=None): This function calculates the mean absolute deviation (MAD) along the specified axis. MAD is a measure of statistical dispersion. The function first calculates the mean of the input data d along the specified axis (or across all axes if axis=None), then it calculates the absolute differences between each element of d and the mean, and finally, it calculates the mean of these absolute differences. It uses NumPy functions for these calculations.\n\n- denoise(x, wavelet='haar', level=1): This function is used for denoising a signal x using wavelet transformation. Here's how it works:\n     - It first decomposes the input signal x into wavelet coefficients using the wavedec function from the PyWavelets (pywt) library. The wavelet used for decomposition is specified by the wavelet parameter, with the default being 'haar' (Haar wavelet), and the decomposition level is specified by the level parameter (default is 1). The decomposition is done in periodic mode (mode=\"per\").\n     - It calculates the standard deviation estimation sigma using the maddest function on the last level of wavelet coefficients (coeff[-level]). This estimation is used for setting the threshold for denoising.\n     - It calculates the threshold value uthresh based on sigma and the length of the signal x.\n     - It applies thresholding to the wavelet coefficients (except the approximation coefficients) using hard thresholding with the calculated threshold value.\n     - It reconstructs the denoised signal using the inverse wavelet transform (waverec function from pywt) on the modified coefficients.\n     - returns the denoised signal.\n     \n\n\n### Loading EEG Data - spectrogram_from_eeg:\n- spectrogram_from_eeg generates spectrograms from EEG\n       - It loads EEG data from a Parquet file specified by parquet_path.\n       - It selects the middle 10,000 samples from the EEG data. This is done by calculating the index of the middle portion of the data and then selecting 10,000 samples around it.\n       \n### Initializing Variables:\n       - It initializes an empty NumPy array img of shape (128, 256, 4) to hold the spectrogram. The first two dimensions represent the height and width of the spectrogram, and the third dimension represents the EEG channels (there are 4 EEG channels).\n       - If display is set to True, it initializes a Matplotlib figure for visualization.\n       \n### Processing EEG Signals:\n   - It iterates over each EEG channel (4 channels in total) and performs the following operations:\n       - Computes pair differences for each EEG channel.\n       - Fills NaN values with the mean of the data.\n       - Optionally, denoises the signal using a wavelet transform if USE_WAVELET is set to True.\n       - Computes the Mel spectrogram of the signal using librosa.feature.melspectrogram.\n       - Performs a log transform on the spectrogram and clips it to a specified width.\n       - Standardizes the spectrogram to have values in the range [-1, 1].\n       - Accumulates the spectrogram values in the img array.\n       \n### Averaging Spectrograms:\n       - After processing all EEG channels, it averages the spectrograms obtained from the four pairs of differences.","metadata":{}},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\n\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n    return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\n\ndef denoise(x, wavelet='haar', level=1):    \n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    sigma = (1/0.6745) * maddest(coeff[-level])\n\n    uthresh = sigma * np.sqrt(2*np.log(len(x)))\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n    ret=pywt.waverec(coeff, wavelet, mode='per')\n    \n    return ret\n\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n    \n    # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n    eeg = pd.read_parquet(parquet_path)\n    middle = (len(eeg)-10_000)//2\n    eeg = eeg.iloc[middle:middle+10_000]\n    \n    # VARIABLE TO HOLD SPECTROGRAM\n    img = np.zeros((128,256,4),dtype='float32')\n    \n    if display: plt.figure(figsize=(10,7))\n    signals = []\n    for k in range(4):\n        COLS = FEATS[k]\n        \n        for kk in range(4):\n        \n            # COMPUTE PAIR DIFFERENCES\n            x = eeg[COLS[kk]].values - eeg[COLS[kk+1]].values\n\n            # FILL NANS\n            m = np.nanmean(x)\n            if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n            else: x[:] = 0\n\n            # DENOISE\n            if USE_WAVELET:\n                x = denoise(x, wavelet=USE_WAVELET)\n            signals.append(x)\n\n            # RAW SPECTROGRAM\n            mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//256, \n                  n_fft=1024, n_mels=128, fmin=0, fmax=20, win_length=128)\n\n            # LOG TRANSFORM\n            width = (mel_spec.shape[1]//32)*32\n            mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n\n            # STANDARDIZE TO -1 TO 1\n            mel_spec_db = (mel_spec_db+40)/40 \n            img[:,:,k] += mel_spec_db\n                \n        # AVERAGE THE 4 MONTAGE DIFFERENCES\n        img[:,:,k] /= 4.0\n        \n        if display:\n            plt.subplot(2,2,k+1)\n            plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n            plt.title(f'EEG {eeg_id} - Spectrogram {NAMES[k]}')\n            \n    if display: \n        plt.show()\n        plt.figure(figsize=(10,5))\n        offset = 0\n        for k in range(4):\n            if k>0: offset -= signals[3-k].min()\n            plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n            offset += signals[3-k].max()\n        plt.legend()\n        plt.title(f'EEG {eeg_id} Signals')\n        plt.show()\n        print(); print('#'*25); print()\n        \n    return img","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:44:35.974209Z","iopub.execute_input":"2024-04-05T12:44:35.974611Z","iopub.status.idle":"2024-04-05T12:44:36.008931Z","shell.execute_reply.started":"2024-04-05T12:44:35.974581Z","shell.execute_reply":"2024-04-05T12:44:36.008062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nDISPLAY = 1\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\n\nprint('Converting Test EEG to Spectrograms...'); print()\nfor i, eeg_id in enumerate(EEG_IDS2):\n        \n    # CREATE SPECTROGRAM FROM EEG PARQUET\n    img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i < DISPLAY)\n    all_eegs2[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:44:48.012478Z","iopub.execute_input":"2024-04-05T12:44:48.013418Z","iopub.status.idle":"2024-04-05T12:44:59.395054Z","shell.execute_reply.started":"2024-04-05T12:44:48.013382Z","shell.execute_reply":"2024-04-05T12:44:59.394118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n- Initialization:\n     - preds = []: Initializes an empty list to store predictions.\n     - test_ds = EEGDataset(test, mode='test', specs=spectrograms2, eeg_specs=all_eegs2): Initializes a dataset object for the testing data. The specifics of EEGDataset custom dataset class for handling EEG data. \n     - test_loader = DataLoader(test_ds, shuffle=False, batch_size=64, num_workers=3): Initializes a DataLoader object for batching and loading the testing data. It will load data from test_ds, with a batch size of 64, no shuffling, and 3 worker processes for loading data.\n     \n- Testing Loop:\n     - A loop runs for 5 iterations, indicating it's likely a 5-fold cross-validation setup.\n     - For each fold:\n          - print('#'*25), print(f'### Testing Fold {i+1}'): Prints out a header for the current fold being tested.\n          - Constructs the checkpoint file name based on the fold index (i) and a variable VER.\n          - Loads a pre-trained EEGEffnetB0 model from the checkpoint file.\n          - Moves the model to the appropriate device (GPU if available) and sets it to evaluation mode (eval()).\n          - Initializes an empty list fold_preds to store predictions for this fold.\n          - Enters a nested loop to iterate through batches of testing data from test_loader.\n          - Moves each batch to the appropriate device.\n          - Performs inference using the model (model(test_batch)), applies softmax to get class probabilities, moves the result to CPU, and converts it to a numpy array.\n          - Appends the predictions for the current batch to fold_preds.\n          - Concatenates the predictions for all batches in this fold (fold_preds) into a numpy array.","metadata":{}},{"cell_type":"code","source":"# INFER EFFICIENTNET ON TEST\npreds = []\ntest_ds = EEGDataset(test, mode='test', specs=spectrograms2, eeg_specs=all_eegs2)\ntest_loader = DataLoader(test_ds, shuffle=False, batch_size=64, num_workers=3)\n\nfor i in range(5):\n    print('#'*25)\n    print(f'### Testing Fold {i+1}')\n\n    ckpt_file = f'EffNet_v{VER}_f{i}.ckpt' if LOAD_MODELS_FROM is None else f'{LOAD_MODELS_FROM}/EffNet_v{VER}_f{i}.ckpt'\n    model = EEGEffnetB0.load_from_checkpoint(ckpt_file)\n    model.to(device).eval()\n    fold_preds = []\n\n    with torch.inference_mode():\n        for test_batch in test_loader:\n            test_batch = test_batch.to(device)\n            pred = torch.softmax(model(test_batch), dim=1).cpu().numpy()\n            fold_preds.append(pred)\n        fold_preds = np.concatenate(fold_preds)\n\n    preds.append(fold_preds)\n\npred = np.mean(preds,axis=0)\nprint()\nprint('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:49:40.412786Z","iopub.execute_input":"2024-04-05T12:49:40.413501Z","iopub.status.idle":"2024-04-05T12:49:51.283605Z","shell.execute_reply.started":"2024-04-05T12:49:40.413467Z","shell.execute_reply":"2024-04-05T12:49:51.282278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'eeg_id': test.eeg_id.values})\nsub[TARGETS] = pred\nsub.to_csv('submission.csv',index=False)\nprint('Submissionn shape',sub.shape)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:55:49.863937Z","iopub.execute_input":"2024-04-05T12:55:49.864725Z","iopub.status.idle":"2024-04-05T12:55:49.889349Z","shell.execute_reply.started":"2024-04-05T12:55:49.864687Z","shell.execute_reply":"2024-04-05T12:55:49.888457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nsub.iloc[:,-6:].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T12:55:52.762363Z","iopub.execute_input":"2024-04-05T12:55:52.762746Z","iopub.status.idle":"2024-04-05T12:55:52.773898Z","shell.execute_reply.started":"2024-04-05T12:55:52.762718Z","shell.execute_reply":"2024-04-05T12:55:52.772935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}