{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7751293,"sourceType":"datasetVersion","datasetId":4456661,"isSourceIdPinned":true}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n\nThis is the inference of this model trained [here](https://www.kaggle.com/code/medali1992/hms-resnet1d-gru-train?scriptVersionId=163239333).\n\n## Version 1\n\n* `CV=0.5162483866506282` `LB=0.48`\n\n### Hyperparams\n\n```\n\n   scheduler='CosineAnnealingWarmRestarts' \n   # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':20,\n        'eta_min':1e-6,\n        'T_mult':1,\n        'last_epoch':-1}\n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_lstm'\n    optimizer='Adam'\n    epochs = 20\n    eps = 1e-6\n    lr = 8e-3\n    min_lr = 1e-6\n    in_channels = 8\n    fc_dim = 512\n    batch_size = 64\n    weight_decay = 1e-3\n    seed = 2024\n```\n## Version 2\n\n* Changed the model architecture\n* `CV=0.5162483866506282` `LB=0.55`\n\n### Hyperparams\n\n```\n\n   scheduler='CosineAnnealingWarmRestarts' \n   # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':20,\n        'eta_min':1e-6,\n        'T_mult':1,\n        'last_epoch':-1}\n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_lstm'\n    optimizer='Adam'\n    epochs = 20\n    eps = 1e-6\n    lr = 8e-3\n    min_lr = 1e-6\n    in_channels = 1\n    batch_size = 32\n    weight_decay = 1e-3\n    seed = 2024\n```\n\n## Version 3\n\n* Added sequence pooling for the rrnn output\nThe Sequence Pooling Layer is used instead of a [CLASS] token in CCTs. This layer introduces a learnable weight which allows the model to perform a weighted average over all the sequences instead of taking output from one special [CLASS] token or from simple average across all the sequences.\nTaken from this [notebook](https://www.kaggle.com/code/utsavnandi/compact-convolutional-transformer-using-pytorch).\n* `CV=0.5238907711166703` `LB=0.49`\n\n```\nclass SeqPool(nn.Module):\n    def __init__(self, emb_dim=192):\n        super().__init__()\n        self.dense = nn.Linear(emb_dim, 1)\n        self.softmax = nn.Softmax(dim=-1)\n\n    def forward(self, x):\n        bs, seq_len, emb_dim = x.shape\n        identity = x\n        x = self.dense(x)\n        x = x.permute(0, 2, 1)\n        x = self.softmax(x)\n        x = x @ identity\n        x = x.reshape(x.shape[0], -1)\n        return x\n```\n\n### Hyperparams\n\n```\n\n   scheduler='CosineAnnealingWarmRestarts' \n   # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':20,\n        'eta_min':1e-6,\n        'T_mult':1,\n        'last_epoch':-1}\n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_lstm'\n    optimizer='Adan'\n    epochs = 20\n    eps = 1e-6\n    lr = 8e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 64\n    weight_decay = 1e-2\n    max_grad_norm = 1e7\n    seed = 2024\n```\n\n## Version 4\n\n* `LB=0.43`\nI divided my data set into two population and trained a two stage model from version1. I took the idea from this [notebook](https://www.kaggle.com/code/seanbearden/effnetb0-2-pop-model-train-twice-lb-0-39/notebook).\n\n### Hyperparams\n\n```\n\n   scheduler='CosineAnnealingWarmRestarts' \n   # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':20,\n        'eta_min':1e-6,\n        'T_mult':1,\n        'last_epoch':-1}\n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_lstm'\n    optimizer='Adan'\n    epochs = 20\n    eps = 1e-6\n    lr = 8e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 64\n    weight_decay = 1e-2\n    max_grad_norm = 1e7\n    seed = 2024\n```\n\n## Version 5\n\n* Changed the CV Scheme to prevent data leakage.\n* `LB=0.44`\n\n```\n\n   scheduler='CosineAnnealingWarmRestarts' \n   # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':20,\n        'eta_min':1e-6,\n        'T_mult':1,\n        'last_epoch':-1}\n    print_freq=50\n    num_workers=1\n    model_name='resnet501d_lstm'\n    optimizer='Adan'\n    stage1_epochs=10\n    stage2_epochs=20\n    eps = 1e-6\n    lr = 8e-3\n    min_lr = 1e-6\n    in_channels=8\n    batch_size=64\n    weight_decay=1e-2\n    max_grad_norm=1e7\n    seed=2024\n```\n\n## Version 6\n\n* I changed the CV sheme, first stage train on all data second stage train on data with total_evaluators >= 10\n\n### Hyperparams\n\n```\n\n   scheduler='OneCycleLR' \n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_gru_transformer'\n    optimizer='Adan'\n    stage1_epochs = 10\n    stage2_epochs = 20\n    eps = 1e-6\n    lr = 1e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 100\n    weight_decay = 1e-2\n    max_grad_norm = 1e7\n    seed = 2024\n```\n\n## Version 7\n\n* I changed the CV sheme, first stage train on all data second stage train on train_pop2\n\n### Hyperparams\n\n```\n\n    scheduler='OneCycleLR' \n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_gru'\n    optimizer='Adan'\n    stage1_epochs = 10\n    stage2_epochs = 10\n    eps = 1e-6\n    lr = 1e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 64\n    weight_decay = 1e-2\n    max_grad_norm = 1e7\n    seed = 2024\n    \n```\n\n## Version 8\n\n* Apply downsampling of factor five\n\n### Hyperparams\n\n```\n\n    scheduler='CosineAnnealingWarmRestarts' \n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_gru'\n    optimizer='Adan'\n    stage1_epochs = 10\n    stage2_epochs = 20\n    eps = 1e-6\n    lr = 1e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 64\n    weight_decay = 1e-2\n    downsample_factor = 5\n    max_grad_norm = 1e7\n    seed = 2024\n    \n```\n\n## Version 9\n\n* Stage2 votes >= 5\n\n### Hyperparams\n\n```\n\n    scheduler='CosineAnnealingWarmRestarts' \n    print_freq=50\n    num_workers = 1\n    model_name = 'resnet501d_gru'\n    optimizer='Adan'\n    stage1_epochs = 10\n    stage2_epochs = 10\n    eps = 1e-6\n    lr = 1e-3\n    min_lr = 1e-6\n    in_channels = 8\n    batch_size = 64\n    weight_decay = 1e-2\n    downsample_factor = 5\n    max_grad_norm = 1e7\n    seed = 2024\n    \n```","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nimport sys\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom typing import Dict, List\nfrom scipy.stats import entropy\nfrom scipy.signal import butter, lfilter, freqz\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\nimport numpy as np\nimport pandas as pd\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score, log_loss\nfrom tqdm.auto import tqdm\nfrom functools import partial\nimport cv2\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport pytorch_lightning as pl\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, OneCycleLR, CosineAnnealingLR, CosineAnnealingWarmRestarts\nfrom sklearn.preprocessing import LabelEncoder\nfrom torchvision.transforms import v2\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import train_test_split\nimport albumentations as A\nfrom albumentations import (Compose, Normalize, Resize, RandomResizedCrop, HorizontalFlip, VerticalFlip, ShiftScaleRotate, Transpose)\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport timm\nimport warnings \nwarnings.filterwarnings('ignore')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nfrom matplotlib import pyplot as plt\nimport joblib\nos.environ['CUDA_VISIBLE_DEVICES'] = \"0,1\"\nVERSION=9","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-03T14:24:49.547469Z","iopub.execute_input":"2024-03-03T14:24:49.547761Z","iopub.status.idle":"2024-03-03T14:25:03.022112Z","shell.execute_reply.started":"2024-03-03T14:24:49.547736Z","shell.execute_reply":"2024-03-03T14:25:03.021221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    PATH = '/kaggle/input/hms-harmful-brain-activity-classification/'\n    test_eeg = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\"\n    test_csv = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"\n    model_name = 'resnet1d_gru'\n    seed = 2024\n    in_channels = 8\n    target_size = 6\n    batch_size = 32\n    num_workers = 1\n\n    \nmodel_weights = [x for x in glob(\"/kaggle/input/resnet1d-gru-weights/pop_2_weight_oof/*.pth\")]\nmodel_weights","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.023653Z","iopub.execute_input":"2024-03-03T14:25:03.023966Z","iopub.status.idle":"2024-03-03T14:25:03.036835Z","shell.execute_reply.started":"2024-03-03T14:25:03.023940Z","shell.execute_reply":"2024-03-03T14:25:03.035805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"def eeg_from_parquet(parquet_path: str) -> np.ndarray:\n    \"\"\"\n    This function reads a parquet file and extracts the middle 50 seconds of readings. Then it fills NaN values\n    with the mean value (ignoring NaNs).\n    :param parquet_path: path to parquet file.\n    :param display: whether to display EEG plots or not.\n    :return data: np.array of shape  (time_steps, eeg_features) -> (10_000, 8)\n    \"\"\"\n    # === Extract middle 50 seconds ===\n    eeg = pd.read_parquet(parquet_path, columns=eeg_features)\n    rows = len(eeg)\n    offset = (rows - 10_000) // 2 # 50 * 200 = 10_000\n    eeg = eeg.iloc[offset:offset+10_000] # middle 50 seconds, has the same amount of readings to left and right\n    # === Convert to numpy ===\n    data = np.zeros((10_000, len(eeg_features))) # create placeholder of same shape with zeros\n    for index, feature in enumerate(eeg_features):\n        x = eeg[feature].values.astype('float32') # convert to float32\n        mean = np.nanmean(x) # arithmetic mean along the specified axis, ignoring NaNs\n        nan_percentage = np.isnan(x).mean() # percentage of NaN values in feature\n        # === Fill nan values ===\n        if nan_percentage < 1: # if some values are nan, but not all\n            x = np.nan_to_num(x, nan=mean)\n        else: # if all values are nan\n            x[:] = 0\n        data[:, index] = x\n   \n    return data\n\n\ndef seed_everything(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed) \n    \n    \ndef sep():\n    print(\"-\"*100)\n\n    \ntarget_preds = [x + \"_pred\" for x in ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']]\nlabel_to_num = {'Seizure': 0, 'LPD': 1, 'GPD': 2, 'LRDA': 3, 'GRDA': 4, 'Other':5}\nnum_to_label = {v: k for k, v in label_to_num.items()}\nseed_everything(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.038322Z","iopub.execute_input":"2024-03-03T14:25:03.038677Z","iopub.status.idle":"2024-03-03T14:25:03.053535Z","shell.execute_reply.started":"2024-03-03T14:25:03.038651Z","shell.execute_reply":"2024-03-03T14:25:03.052503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(CFG.test_csv)\nprint(f\"Test dataframe shape is: {test_df.shape}\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.056105Z","iopub.execute_input":"2024-03-03T14:25:03.056402Z","iopub.status.idle":"2024-03-03T14:25:03.083978Z","shell.execute_reply.started":"2024-03-03T14:25:03.056378Z","shell.execute_reply":"2024-03-03T14:25:03.083004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eeg_parquet_paths = glob(CFG.test_eeg+ \"*.parquet\")\neeg_df = pd.read_parquet(eeg_parquet_paths[0])\neeg_features = eeg_df.columns\nprint(f'There are {len(eeg_features)} raw eeg features')\nprint(list(eeg_features))\neeg_features = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\nfeature_to_index = {x:y for x,y in zip(eeg_features, range(len(eeg_features)))}","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.085114Z","iopub.execute_input":"2024-03-03T14:25:03.085398Z","iopub.status.idle":"2024-03-03T14:25:03.284496Z","shell.execute_reply.started":"2024-03-03T14:25:03.085373Z","shell.execute_reply":"2024-03-03T14:25:03.283284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nCREATE_EEGS = False\nall_eegs = {}\nvisualize = 1\neeg_paths = glob(CFG.test_eeg + \"*.parquet\")\neeg_ids = test_df.eeg_id.unique()\n\nfor i, eeg_id in tqdm(enumerate(eeg_ids)):  \n    # Save EEG to Python dictionary of numpy arrays\n    eeg_path = CFG.test_eeg + str(eeg_id) + \".parquet\"\n    data = eeg_from_parquet(eeg_path)              \n    all_eegs[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.285713Z","iopub.execute_input":"2024-03-03T14:25:03.285979Z","iopub.status.idle":"2024-03-03T14:25:03.323357Z","shell.execute_reply.started":"2024-03-03T14:25:03.285957Z","shell.execute_reply":"2024-03-03T14:25:03.322391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import butter, lfilter\n\ndef butter_lowpass_filter(data, cutoff_freq: int = 20, sampling_rate: int = 200, order: int = 4):\n    nyquist = 0.5 * sampling_rate\n    normal_cutoff = cutoff_freq / nyquist\n    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n    filtered_data = lfilter(b, a, data, axis=0)\n    return filtered_data","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:25:03.324560Z","iopub.execute_input":"2024-03-03T14:25:03.324847Z","iopub.status.idle":"2024-03-03T14:25:03.330748Z","shell.execute_reply.started":"2024-03-03T14:25:03.324822Z","shell.execute_reply":"2024-03-03T14:25:03.329830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class EEGDataset(Dataset):\n    def __init__(\n        self, df: pd.DataFrame, config, mode: str = 'train',\n        eegs: Dict[int, np.ndarray] = all_eegs, downsample: int = None\n    ): \n        self.df = df\n        self.config = config\n        self.mode = mode\n        self.eegs = eegs\n        self.downsample = downsample\n        \n    def __len__(self):\n        \"\"\"\n        Length of dataset.\n        \"\"\"\n        return len(self.df)\n        \n    def __getitem__(self, index):\n        \"\"\"\n        Get one item.\n        \"\"\"\n        X, y = self.__data_generation(index)\n        if self.downsample is not None:\n            X = X[::self.downsample,:]\n        output = {\n            \"eeg\": torch.tensor(X, dtype=torch.float32),\n            \"labels\": torch.tensor(y, dtype=torch.float32)\n        }\n        return output\n                        \n    def __data_generation(self, index):\n        row = self.df.iloc[index]\n        X = np.zeros((10_000, 8), dtype='float32')\n        y = np.zeros(6, dtype='float32')\n        data = self.eegs[row.eeg_id]\n\n        # === Feature engineering ===\n        X[:,0] = data[:,feature_to_index['Fp1']] - data[:,feature_to_index['T3']]\n        X[:,1] = data[:,feature_to_index['T3']] - data[:,feature_to_index['O1']]\n\n        X[:,2] = data[:,feature_to_index['Fp1']] - data[:,feature_to_index['C3']]\n        X[:,3] = data[:,feature_to_index['C3']] - data[:,feature_to_index['O1']]\n\n        X[:,4] = data[:,feature_to_index['Fp2']] - data[:,feature_to_index['C4']]\n        X[:,5] = data[:,feature_to_index['C4']] - data[:,feature_to_index['O2']]\n\n        X[:,6] = data[:,feature_to_index['Fp2']] - data[:,feature_to_index['T4']]\n        X[:,7] = data[:,feature_to_index['T4']] - data[:,feature_to_index['O2']]\n\n        # === Standarize ===\n        X = np.clip(X,-1024, 1024)\n        X = np.nan_to_num(X, nan=0) / 32.0\n\n        # === Butter Low-pass Filter ===\n        X = butter_lowpass_filter(X)\n        if self.mode != 'test':\n            y = row[self.config.target_cols].values.astype(np.float32)\n            \n        return X, y","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:27:35.828560Z","iopub.execute_input":"2024-03-03T14:27:35.828946Z","iopub.status.idle":"2024-03-03T14:27:35.845495Z","shell.execute_reply.started":"2024-03-03T14:27:35.828907Z","shell.execute_reply":"2024-03-03T14:27:35.844320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataLoader","metadata":{}},{"cell_type":"code","source":"test_dataset = EEGDataset(test_df, CFG, mode='test')\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=CFG.batch_size,\n    shuffle=False,\n    num_workers=CFG.num_workers,\n    pin_memory=True,\n    drop_last=False\n)\noutput = test_dataset[0]\nX = output[\"eeg\"]\nprint(f\"X shape: {X.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:27:36.370393Z","iopub.execute_input":"2024-03-03T14:27:36.370765Z","iopub.status.idle":"2024-03-03T14:27:36.383047Z","shell.execute_reply.started":"2024-03-03T14:27:36.370740Z","shell.execute_reply":"2024-03-03T14:27:36.382157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class ResNet_1D_Block(nn.Module):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, downsampling):\n        super(ResNet_1D_Block, self).__init__()\n        self.bn1 = nn.BatchNorm1d(num_features=in_channels)\n        self.relu = nn.ReLU(inplace=False)\n        self.dropout = nn.Dropout(p=0.0, inplace=False)\n        self.conv1 = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n        self.conv2 = nn.Conv1d(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.maxpool = nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n        self.downsampling = downsampling\n\n    def forward(self, x):\n        identity = x\n\n        out = self.bn1(x)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv1(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv2(out)\n\n        out = self.maxpool(out)\n        identity = self.downsampling(x)\n\n        out += identity\n        return out\n\n\nclass EEGNet(nn.Module):\n\n    def __init__(self, kernels, in_channels=20, fixed_kernel_size=17, num_classes=6):\n        super(EEGNet, self).__init__()\n        self.kernels = kernels\n        self.planes = 24\n        self.parallel_conv = nn.ModuleList()\n        self.in_channels = in_channels\n        \n        for i, kernel_size in enumerate(list(self.kernels)):\n            sep_conv = nn.Conv1d(in_channels=in_channels, out_channels=self.planes, kernel_size=(kernel_size),\n                               stride=1, padding=0, bias=False,)\n            self.parallel_conv.append(sep_conv)\n\n        self.bn1 = nn.BatchNorm1d(num_features=self.planes)\n        self.relu = nn.ReLU(inplace=False)\n        self.conv1 = nn.Conv1d(in_channels=self.planes, out_channels=self.planes, kernel_size=fixed_kernel_size,\n                               stride=2, padding=2, bias=False)\n        self.block = self._make_resnet_layer(kernel_size=fixed_kernel_size, stride=1, padding=fixed_kernel_size//2)\n        self.bn2 = nn.BatchNorm1d(num_features=self.planes)\n        self.avgpool = nn.AvgPool1d(kernel_size=6, stride=6, padding=2)\n        self.rnn = nn.GRU(input_size=self.in_channels, hidden_size=128, num_layers=1, bidirectional=True)\n        self.fc = nn.Linear(in_features=424, out_features=num_classes)\n\n    def _make_resnet_layer(self, kernel_size, stride, blocks=9, padding=0):\n        layers = []\n        downsample = None\n        base_width = self.planes\n\n        for i in range(blocks):\n            downsampling = nn.Sequential(\n                    nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n                )\n            layers.append(ResNet_1D_Block(in_channels=self.planes, out_channels=self.planes, kernel_size=kernel_size,\n                                       stride=stride, padding=padding, downsampling=downsampling))\n\n        return nn.Sequential(*layers)\n    def extract_features(self, x):\n        x = x.permute(0, 2, 1)\n        out_sep = []\n\n        for i in range(len(self.kernels)):\n            sep = self.parallel_conv[i](x)\n            out_sep.append(sep)\n\n        out = torch.cat(out_sep, dim=2)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv1(out)  \n\n        out = self.block(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.avgpool(out)  \n        \n        out = out.reshape(out.shape[0], -1)  \n        rnn_out, _ = self.rnn(x.permute(0, 2, 1))\n        new_rnn_h = rnn_out[:, -1, :]  \n\n        new_out = torch.cat([out, new_rnn_h], dim=1) \n        return new_out\n    \n    def forward(self, x):\n        new_out = self.extract_features(x)\n        result = self.fc(new_out)  \n\n        return result","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:27:38.933084Z","iopub.execute_input":"2024-03-03T14:27:38.933410Z","iopub.status.idle":"2024-03-03T14:27:38.956984Z","shell.execute_reply.started":"2024-03-03T14:27:38.933386Z","shell.execute_reply":"2024-03-03T14:27:38.956051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Function","metadata":{}},{"cell_type":"code","source":"def inference_function(test_loader, model, device):\n    model.eval() # set model in evaluation mode\n    softmax = nn.Softmax(dim=1)\n    prediction_dict = {}\n    preds = []\n    with tqdm(test_loader, unit=\"test_batch\", desc='Inference') as tqdm_test_loader:\n        for step, batch in enumerate(tqdm_test_loader):\n            X = batch.pop(\"eeg\").to(device) # send inputs to `device`\n            batch_size = X.size(0)\n            with torch.no_grad():\n                y_preds = model(X) # forward propagation pass\n            y_preds = softmax(y_preds)\n            preds.append(y_preds.to('cpu').numpy()) # save predictions\n                \n    prediction_dict[\"predictions\"] = np.concatenate(preds) # np.array() of shape (fold_size, target_cols)\n    return prediction_dict","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:27:39.551885Z","iopub.execute_input":"2024-03-03T14:27:39.552529Z","iopub.status.idle":"2024-03-03T14:27:39.559988Z","shell.execute_reply.started":"2024-03-03T14:27:39.552494Z","shell.execute_reply":"2024-03-03T14:27:39.558881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference ","metadata":{}},{"cell_type":"code","source":"predictions = []\n\nfor model_weight in model_weights:\n    test_dataset = EEGDataset(test_df, CFG, mode='test')\n    train_loader = DataLoader(\n        test_dataset,\n        batch_size=CFG.batch_size,\n        shuffle=False,\n        num_workers=CFG.num_workers,\n        pin_memory=True,\n        drop_last=False\n    )\n    model = EEGNet(kernels=[3,5,7,9], in_channels=CFG.in_channels, fixed_kernel_size=5, num_classes=CFG.target_size)\n    checkpoint = torch.load(model_weight, map_location=device)\n    model.load_state_dict(checkpoint[\"model\"])\n    model.to(device)\n    prediction_dict = inference_function(test_loader, model, device)\n    predictions.append(prediction_dict[\"predictions\"])\n    torch.cuda.empty_cache()\n    gc.collect()\n    \npredictions = np.array(predictions)\npredictions = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:27:40.484169Z","iopub.execute_input":"2024-03-03T14:27:40.485044Z","iopub.status.idle":"2024-03-03T14:28:13.349218Z","shell.execute_reply.started":"2024-03-03T14:27:40.485010Z","shell.execute_reply":"2024-03-03T14:28:13.347989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nsub = pd.DataFrame({'eeg_id': test_df.eeg_id.values})\nsub[TARGETS] = predictions\nsub.to_csv(f'submission.csv',index=False)\nprint(f'Submission shape: {sub.shape}')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T14:28:17.548102Z","iopub.execute_input":"2024-03-03T14:28:17.548892Z","iopub.status.idle":"2024-03-03T14:28:17.576575Z","shell.execute_reply.started":"2024-03-03T14:28:17.548855Z","shell.execute_reply":"2024-03-03T14:28:17.575482Z"},"trusted":true},"execution_count":null,"outputs":[]}]}