{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n dataset = os.path.join(dirname, filename)\n print(dataset)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-06T21:02:07.003996Z","iopub.execute_input":"2024-04-06T21:02:07.004314Z","iopub.status.idle":"2024-04-06T21:02:12.142822Z","shell.execute_reply.started":"2024-04-06T21:02:07.004289Z","shell.execute_reply":"2024-04-06T21:02:12.141834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:00:23.565427Z","iopub.execute_input":"2024-04-06T21:00:23.565725Z","iopub.status.idle":"2024-04-06T21:00:23.570915Z","shell.execute_reply.started":"2024-04-06T21:00:23.565702Z","shell.execute_reply":"2024-04-06T21:00:23.569857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport datetime as dt\nimport numpy as np\nimport pandas as pd\n\nfrom glob import glob\nfrom pathlib import Path\nfrom typing import Dict, List, Union\nfrom scipy.signal import butter, lfilter, freqz\nfrom matplotlib import pyplot as plt\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\nsys.path.append(\"/kaggle/input/kaggle-kl-div\")\nfrom kaggle_kl_div import score\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\ndevice = torch.device(\"cuda\")\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n\n!cat /etc/os-release | grep -oP \"PRETTY_NAME=\\\"\\K([^\\\"]*)\"\nprint(f\"BUILD_DATE={os.environ['BUILD_DATE']}, CONTAINER_NAME={os.environ['CONTAINER_NAME']}\")\n\ntry:\n print(\n f\"PyTorch Version:{torch.__version__}, CUDA is available:{torch.cuda.is_available()}, Version CUDA:{torch.version.cuda}\"\n )\n print(\n f\"Device Capability:{torch.cuda.get_device_capability()}, {torch.cuda.get_arch_list()}\"\n )\n print(\n f\"CuDNN Enabled:{torch.backends.cudnn.enabled}, Version:{torch.backends.cudnn.version()}\"\n )\nexcept Exception:\n pass","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:06:20.893523Z","iopub.execute_input":"2024-04-06T21:06:20.893913Z","iopub.status.idle":"2024-04-06T21:06:21.492631Z","shell.execute_reply.started":"2024-04-06T21:06:20.893883Z","shell.execute_reply":"2024-04-06T21:06:21.491550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n VERSION = 88\n\n model_name = \"resnet1d_gru\"\n\n seed = 2024\n batch_size = 32\n num_workers = 0\n\n fixed_kernel_size = 5\n # kernels = [3, 5, 7, 9]\n # linear_layer_features = 424\n kernels = [3, 5, 7, 9, 11]\n #linear_layer_features = 448 # Full Signal = 10_000\n #linear_layer_features = 352 # Half Signal = 5_000\n linear_layer_features = 304 # 1/5 Signal = 2_000\n\n seq_length = 50 # Second's\n sampling_rate = 200 # Hz\n nsamples = seq_length * sampling_rate # Число семплов\n out_samples = nsamples // 5\n\n # bandpass_filter = {\"low\": 0.5, \"high\": 20, \"order\": 2}\n # rand_filter = {\"probab\": 0.1, \"low\": 10, \"high\": 20, \"band\": 1.0, \"order\": 2}\n freq_channels = [] # [(8.0, 12.0)]; [(0.5, 4.5)]\n filter_order = 2\n random_close_zone = 0.0 # 0.2\n \n target_cols = [\n \"seizure_vote\",\n \"lpd_vote\",\n \"gpd_vote\",\n \"lrda_vote\",\n \"grda_vote\",\n \"other_vote\",\n ]\n\n # target_preds = [x + \"_pred\" for x in target_cols]\n # label_to_num = {\"Seizure\": 0, \"LPD\": 1, \"GPD\": 2, \"LRDA\": 3, \"GRDA\": 4, \"Other\": 5}\n # num_to_label = {v: k for k, v in label_to_num.items()}\n\n map_features = [\n (\"Fp1\", \"T3\"),\n (\"T3\", \"O1\"),\n (\"Fp1\", \"C3\"),\n (\"C3\", \"O1\"),\n (\"Fp2\", \"C4\"),\n (\"C4\", \"O2\"),\n (\"Fp2\", \"T4\"),\n (\"T4\", \"O2\"),\n #('Fz', 'Cz'), ('Cz', 'Pz'), \n ]\n\n eeg_features = [\"Fp1\", \"T3\", \"C3\", \"O1\", \"Fp2\", \"C4\", \"T4\", \"O2\"] # 'Fz', 'Cz', 'Pz']\n # 'F3', 'P3', 'F7', 'T5', 'Fz', 'Cz', 'Pz', 'F4', 'P4', 'F8', 'T6', 'EKG'] \n feature_to_index = {x: y for x, y in zip(eeg_features, range(len(eeg_features)))}\n simple_features = [] # 'Fz', 'Cz', 'Pz', 'EKG'\n\n # eeg_features = [row for row in feature_to_index]\n # eeg_feat_size = len(eeg_features)\n \n n_map_features = len(map_features)\n in_channels = n_map_features + n_map_features * len(freq_channels) + len(simple_features)\n target_size = len(target_cols)\n \n PATH = \"/kaggle/input/hms-harmful-brain-activity-classification/\"\n test_eeg = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\"\n test_csv = \"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\"","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:07:23.916520Z","iopub.execute_input":"2024-04-06T21:07:23.916975Z","iopub.status.idle":"2024-04-06T21:07:23.928668Z","shell.execute_reply.started":"2024-04-06T21:07:23.916944Z","shell.execute_reply":"2024-04-06T21:07:23.927616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"koef_1 = 1.0\nmodel_weights = [\n {\n 'bandpass_filter':{'low':0.5, 'high':20, 'order':2}, \n 'file_data': \n [\n #{'koef':koef_1, 'file_mask':\"/kaggle/input/hms-resnet1d-gru-weights-v82/pop_1_weight_oof/*_best.pth\"},\n {'koef':koef_1, 'file_mask':\"/kaggle/input/hms-resnet1d-gru-weights-v82/pop_2_weight_oof/*_best.pth\"},\n ]\n },\n]","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:07:58.046605Z","iopub.execute_input":"2024-04-06T21:07:58.046963Z","iopub.status.idle":"2024-04-06T21:07:58.052317Z","shell.execute_reply.started":"2024-04-06T21:07:58.046939Z","shell.execute_reply":"2024-04-06T21:07:58.051450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_logger(log_file=\"./test.log\"):\n from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n\n logger = getLogger(__name__)\n logger.setLevel(INFO)\n handler1 = StreamHandler()\n handler1.setFormatter(Formatter(\"%(message)s\"))\n handler2 = FileHandler(filename=log_file)\n handler2.setFormatter(Formatter(\"%(message)s\"))\n logger.addHandler(handler1)\n logger.addHandler(handler2)\n return logger\n\n\ndef asMinutes(s):\n m = math.floor(s / 60)\n s -= m * 60\n return \"%dm %ds\" % (m, s)\n\n\ndef timeSince(since, percent):\n now = time.time()\n s = now - since\n es = s / (percent)\n rs = es - s\n return \"%s (remain %s)\" % (asMinutes(s), asMinutes(rs))\n\n\ndef quantize_data(data, classes):\n mu_x = mu_law_encoding(data, classes)\n return mu_x # quantized\n\n\ndef mu_law_encoding(data, mu):\n mu_x = np.sign(data) * np.log(1 + mu * np.abs(data)) / np.log(mu + 1)\n return mu_x\n\n\ndef mu_law_expansion(data, mu):\n s = np.sign(data) * (np.exp(np.abs(data) * np.log(mu + 1)) - 1) / mu\n return s\n\n\ndef butter_bandpass(lowcut, highcut, fs, order=5):\n return butter(order, [lowcut, highcut], fs=fs, btype=\"band\")\n\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n y = lfilter(b, a, data)\n return y\n\n\ndef butter_lowpass_filter(\n data, cutoff_freq=20, sampling_rate=CFG.sampling_rate, order=4\n):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n b, a = butter(order, normal_cutoff, btype=\"low\", analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n return filtered_data\n\n\ndef denoise_filter(x):\n y = butter_bandpass_filter(x, CFG.lowcut, CFG.highcut, CFG.sampling_rate, order=6)\n y = (y + np.roll(y, -1) + np.roll(y, -2) + np.roll(y, -3)) / 4\n y = y[0:-1:4]\n return y","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:08:21.428740Z","iopub.execute_input":"2024-04-06T21:08:21.429094Z","iopub.status.idle":"2024-04-06T21:08:21.445485Z","shell.execute_reply.started":"2024-04-06T21:08:21.429067Z","shell.execute_reply":"2024-04-06T21:08:21.444707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eeg_from_parquet(\n parquet_path: str, display: bool = False, seq_length=CFG.seq_length\n) -> np.ndarray:\n \"\"\"\n Эта функция читает файл паркета и извлекает средние 50 секунд показаний. Затем он заполняет значения NaN\n со средним значением (игнорируя NaN).\n :param parquet_path: путь к файлу паркета.\n :param display: отображать графики ЭЭГ или нет.\n :return data: np.array формы (time_steps, eeg_features) -> (10_000, 8)\n \"\"\"\n\n # Вырезаем среднюю 50 секундную часть\n eeg = pd.read_parquet(parquet_path, columns=CFG.eeg_features)\n rows = len(eeg)\n\n # начало смещения данных, чтобы забрать середину\n offset = (rows - CFG.nsamples) // 2\n\n # средние 50 секунд, имеет одинаковое количество показаний слева и справа\n eeg = eeg.iloc[offset : offset + CFG.nsamples]\n\n if display:\n plt.figure(figsize=(10, 5))\n offset = 0\n\n # Конвертировать в numpy\n\n # создать заполнитель той же формы с нулями\n data = np.zeros((CFG.nsamples, len(CFG.eeg_features)))\n\n for index, feature in enumerate(CFG.eeg_features):\n x = eeg[feature].values.astype(\"float32\") # конвертировать в float32\n\n # Вычисляет среднее арифметическое вдоль указанной оси, игнорируя NaN.\n mean = np.nanmean(x)\n nan_percentage = np.isnan(x).mean() # percentage of NaN values in feature\n\n # Заполнение значения Nan\n # Поэлементная проверка на NaN и возврат результата в виде логического массива.\n if nan_percentage < 1: # если некоторые значения равны Nan, но не все\n x = np.nan_to_num(x, nan=mean)\n else: # если все значения — Nan\n x[:] = 0\n data[:, index] = x\n\n if display:\n if index != 0:\n offset += x.max()\n plt.plot(range(CFG.nsamples), x - offset, label=feature)\n offset -= x.min()\n\n if display:\n plt.legend()\n name = parquet_path.split(\"/\")[-1].split(\".\")[0]\n plt.yticks([])\n plt.title(f\"EEG {name}\", size=16)\n plt.show()\n return data","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:09:11.890381Z","iopub.execute_input":"2024-04-06T21:09:11.890970Z","iopub.status.idle":"2024-04-06T21:09:11.903267Z","shell.execute_reply.started":"2024-04-06T21:09:11.890932Z","shell.execute_reply":"2024-04-06T21:09:11.902203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGDataset(Dataset):\n def __init__(\n self,\n df: pd.DataFrame,\n batch_size: int,\n eegs: Dict[int, np.ndarray],\n mode: str = \"train\",\n downsample: int = None,\n bandpass_filter: Dict[str, Union[int, float]] = None,\n rand_filter: Dict[str, Union[int, float]] = None,\n ):\n self.df = df\n self.batch_size = batch_size\n self.mode = mode\n self.eegs = eegs\n self.downsample = downsample\n self.bandpass_filter = bandpass_filter\n self.rand_filter = rand_filter\n \n def __len__(self):\n \"\"\"\n Length of dataset.\n \"\"\"\n # Обозначает количество пакетов за эпоху\n return len(self.df)\n\n def __getitem__(self, index):\n \"\"\"\n Get one item.\n \"\"\"\n # Сгенерировать один пакет данных\n X, y_prob = self.__data_generation(index)\n if self.downsample is not None:\n X = X[:: self.downsample, :]\n output = {\n \"eeg\": torch.tensor(X, dtype=torch.float32),\n \"labels\": torch.tensor(y_prob, dtype=torch.float32),\n }\n return output\n\n def __data_generation(self, index):\n # Генерирует данные, содержащие образцы размера партии\n X = np.zeros(\n (CFG.out_samples, CFG.in_channels), dtype=\"float32\"\n ) # Size=(10000, 14)\n\n row = self.df.iloc[index] # Строка Pandas\n data = self.eegs[row.eeg_id] # Size=(10000, 8)\n if CFG.nsamples != CFG.out_samples:\n if self.mode != \"train\":\n offset = (CFG.nsamples - CFG.out_samples) // 2\n else:\n #offset = random.randint(0, CFG.nsamples - CFG.out_samples) \n offset = ((CFG.nsamples - CFG.out_samples) * random.randint(0, 1000)) // 1000\n data = data[offset:offset+CFG.out_samples,:]\n\n for i, (feat_a, feat_b) in enumerate(CFG.map_features):\n if self.mode == \"train\" and CFG.random_close_zone > 0 and random.uniform(0.0, 1.0) <= CFG.random_close_zone:\n continue\n \n diff_feat = (\n data[:, CFG.feature_to_index[feat_a]]\n - data[:, CFG.feature_to_index[feat_b]]\n ) # Size=(10000,)\n\n if not self.bandpass_filter is None:\n diff_feat = butter_bandpass_filter(\n diff_feat,\n self.bandpass_filter[\"low\"],\n self.bandpass_filter[\"high\"],\n CFG.sampling_rate,\n order=self.bandpass_filter[\"order\"],\n )\n \n if (\n self.mode == \"train\"\n and not self.rand_filter is None\n and random.uniform(0.0, 1.0) <= self.rand_filter[\"probab\"]\n ):\n lowcut = random.randint(\n self.rand_filter[\"low\"], self.rand_filter[\"high\"]\n )\n highcut = lowcut + self.rand_filter[\"band\"]\n diff_feat = butter_bandpass_filter(\n diff_feat,\n lowcut,\n highcut,\n CFG.sampling_rate,\n order=self.rand_filter[\"order\"],\n )\n\n X[:, i] = diff_feat\n\n n = CFG.n_map_features\n if len(CFG.freq_channels) > 0:\n for i in range(CFG.n_map_features):\n diff_feat = X[:, i]\n for j, (lowcut, highcut) in enumerate(CFG.freq_channels):\n band_feat = butter_bandpass_filter(\n diff_feat, lowcut, highcut, CFG.sampling_rate, order=CFG.filter_order, # 6\n )\n X[:, n] = band_feat\n n += 1\n\n for spml_feat in CFG.simple_features:\n feat_val = data[:, CFG.feature_to_index[spml_feat]]\n \n if not self.bandpass_filter is None:\n feat_val = butter_bandpass_filter(\n feat_val,\n self.bandpass_filter[\"low\"],\n self.bandpass_filter[\"high\"],\n CFG.sampling_rate,\n order=self.bandpass_filter[\"order\"],\n )\n\n if (\n self.mode == \"train\"\n and not self.rand_filter is None\n and random.uniform(0.0, 1.0) <= self.rand_filter[\"probab\"]\n ):\n lowcut = random.randint(\n self.rand_filter[\"low\"], self.rand_filter[\"high\"]\n )\n highcut = lowcut + self.rand_filter[\"band\"]\n feat_val = butter_bandpass_filter(\n feat_val,\n lowcut,\n highcut,\n CFG.sampling_rate,\n order=self.rand_filter[\"order\"],\n )\n\n X[:, n] = feat_val\n n += 1\n \n # Обрезать края превышающие значения [-1024, 1024]\n X = np.clip(X, -1024, 1024)\n\n # Замените NaN нулем и разделить все на 32\n X = np.nan_to_num(X, nan=0) / 32.0\n\n # обрезать полосовым фильтром верхнюю границу в 20 Hz.\n X = butter_lowpass_filter(X, order=CFG.filter_order) # 4\n\n y_prob = np.zeros(CFG.target_size, dtype=\"float32\") # Size=(6,)\n if self.mode != \"test\":\n y_prob = row[CFG.target_cols].values.astype(np.float32)\n\n return X, y_prob","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:09:38.129561Z","iopub.execute_input":"2024-04-06T21:09:38.129917Z","iopub.status.idle":"2024-04-06T21:09:38.154734Z","shell.execute_reply.started":"2024-04-06T21:09:38.129895Z","shell.execute_reply":"2024-04-06T21:09:38.153892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet_1D_Block(nn.Module):\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride,\n padding,\n downsampling,\n dilation=1,\n groups=1,\n dropout=0.0,\n ):\n super(ResNet_1D_Block, self).__init__()\n\n self.bn1 = nn.BatchNorm1d(num_features=in_channels)\n # self.relu = nn.ReLU(inplace=False)\n # self.relu_1 = nn.PReLU()\n # self.relu_2 = nn.PReLU()\n self.relu_1 = nn.Hardswish()\n self.relu_2 = nn.Hardswish()\n\n self.dropout = nn.Dropout(p=dropout, inplace=False)\n self.conv1 = nn.Conv1d(\n in_channels=in_channels,\n out_channels=out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n self.conv2 = nn.Conv1d(\n in_channels=out_channels,\n out_channels=out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.maxpool = nn.MaxPool1d(\n kernel_size=2,\n stride=2,\n padding=0,\n dilation=dilation,\n )\n self.downsampling = downsampling\n\n def forward(self, x):\n identity = x\n\n out = self.bn1(x)\n out = self.relu_1(out)\n out = self.dropout(out)\n out = self.conv1(out)\n out = self.bn2(out)\n out = self.relu_2(out)\n out = self.dropout(out)\n out = self.conv2(out)\n\n out = self.maxpool(out)\n identity = self.downsampling(x)\n\n out += identity\n return out\n\n\nclass EEGNet(nn.Module):\n def __init__(\n self,\n kernels,\n in_channels,\n fixed_kernel_size,\n num_classes,\n linear_layer_features,\n dilation=1,\n groups=1,\n ):\n super(EEGNet, self).__init__()\n self.kernels = kernels\n self.planes = 24\n self.parallel_conv = nn.ModuleList()\n self.in_channels = in_channels\n\n for i, kernel_size in enumerate(list(self.kernels)):\n sep_conv = nn.Conv1d(\n in_channels=in_channels,\n out_channels=self.planes,\n kernel_size=(kernel_size),\n stride=1,\n padding=0,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n self.parallel_conv.append(sep_conv)\n\n self.bn1 = nn.BatchNorm1d(num_features=self.planes)\n # self.relu = nn.ReLU(inplace=False)\n # self.relu_1 = nn.ReLU()\n # self.relu_2 = nn.ReLU()\n self.relu_1 = nn.SiLU()\n self.relu_2 = nn.SiLU()\n\n self.conv1 = nn.Conv1d(\n in_channels=self.planes,\n out_channels=self.planes,\n kernel_size=fixed_kernel_size,\n stride=2,\n padding=2,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.block = self._make_resnet_layer(\n kernel_size=fixed_kernel_size,\n stride=1,\n dilation=dilation,\n groups=groups,\n padding=fixed_kernel_size // 2,\n )\n self.bn2 = nn.BatchNorm1d(num_features=self.planes)\n self.avgpool = nn.AvgPool1d(kernel_size=6, stride=6, padding=2)\n\n self.rnn = nn.GRU(\n input_size=self.in_channels,\n hidden_size=128,\n num_layers=1,\n bidirectional=True,\n # dropout=0.2,\n )\n\n self.fc = nn.Linear(in_features=linear_layer_features, out_features=num_classes)\n\n def _make_resnet_layer(\n self,\n kernel_size,\n stride,\n dilation=1,\n groups=1,\n blocks=9,\n padding=0,\n dropout=0.0,\n ):\n layers = []\n downsample = None\n base_width = self.planes\n\n for i in range(blocks):\n downsampling = nn.Sequential(\n nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n )\n layers.append(\n ResNet_1D_Block(\n in_channels=self.planes,\n out_channels=self.planes,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n downsampling=downsampling,\n dilation=dilation,\n groups=groups,\n dropout=dropout,\n )\n )\n return nn.Sequential(*layers)\n\n def extract_features(self, x):\n x = x.permute(0, 2, 1)\n out_sep = []\n\n for i in range(len(self.kernels)):\n sep = self.parallel_conv[i](x)\n out_sep.append(sep)\n\n out = torch.cat(out_sep, dim=2)\n out = self.bn1(out)\n out = self.relu_1(out)\n out = self.conv1(out)\n\n out = self.block(out)\n out = self.bn2(out)\n out = self.relu_2(out)\n out = self.avgpool(out)\n\n out = out.reshape(out.shape[0], -1)\n rnn_out, _ = self.rnn(x.permute(0, 2, 1))\n new_rnn_h = rnn_out[:, -1, :] # <~~\n\n new_out = torch.cat([out, new_rnn_h], dim=1)\n return new_out\n\n def forward(self, x):\n new_out = self.extract_features(x)\n result = self.fc(new_out)\n return result","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:10:05.673073Z","iopub.execute_input":"2024-04-06T21:10:05.673404Z","iopub.status.idle":"2024-04-06T21:10:05.699735Z","shell.execute_reply.started":"2024-04-06T21:10:05.673368Z","shell.execute_reply":"2024-04-06T21:10:05.698782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_function(test_loader, model, device):\n model.eval() # set model in evaluation mode\n softmax = nn.Softmax(dim=1)\n prediction_dict = {}\n preds = []\n with tqdm(test_loader, unit=\"test_batch\", desc=\"Inference\") as tqdm_test_loader:\n for step, batch in enumerate(tqdm_test_loader):\n X = batch.pop(\"eeg\").to(device) # send inputs to `device`\n batch_size = X.size(0)\n with torch.no_grad():\n y_preds = model(X) # forward propagation pass\n y_preds = softmax(y_preds)\n preds.append(y_preds.to(\"cpu\").numpy()) # save predictions\n\n prediction_dict[\"predictions\"] = np.concatenate(\n preds\n ) # np.array() of shape (fold_size, target_cols)\n return prediction_dict","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:10:21.645252Z","iopub.execute_input":"2024-04-06T21:10:21.645626Z","iopub.status.idle":"2024-04-06T21:10:21.652918Z","shell.execute_reply.started":"2024-04-06T21:10:21.645597Z","shell.execute_reply":"2024-04-06T21:10:21.651958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(CFG.test_csv)\nprint(f\"Test dataframe shape is: {test_df.shape}\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:10:33.104993Z","iopub.execute_input":"2024-04-06T21:10:33.105457Z","iopub.status.idle":"2024-04-06T21:10:33.130204Z","shell.execute_reply.started":"2024-04-06T21:10:33.105420Z","shell.execute_reply":"2024-04-06T21:10:33.129348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_eeg_parquet_paths = glob(CFG.test_eeg + \"*.parquet\")\ntest_eeg_df = pd.read_parquet(test_eeg_parquet_paths[0])\ntest_eeg_features = test_eeg_df.columns\nprint(f\"There are {len(test_eeg_features)} raw eeg features\")\nprint(list(test_eeg_features))\ndel test_eeg_df\n_ = gc.collect()\n\n# %%time\nall_eegs = {}\neeg_ids = test_df.eeg_id.unique()\nfor i, eeg_id in tqdm(enumerate(eeg_ids)):\n # Save EEG to Python dictionary of numpy arrays\n eeg_path = CFG.test_eeg + str(eeg_id) + \".parquet\"\n data = eeg_from_parquet(eeg_path)\n all_eegs[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:10:47.197051Z","iopub.execute_input":"2024-04-06T21:10:47.197417Z","iopub.status.idle":"2024-04-06T21:10:47.644545Z","shell.execute_reply.started":"2024-04-06T21:10:47.197376Z","shell.execute_reply":"2024-04-06T21:10:47.643679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"koef_sum = 0\nkoef_count = 0\npredictions = []\nfiles = []\n \nfor model_block in model_weights:\n test_dataset = EEGDataset(\n df=test_df,\n batch_size=CFG.batch_size,\n mode=\"test\",\n eegs=all_eegs,\n bandpass_filter=model_block['bandpass_filter']\n )\n\n if len(predictions) == 0:\n output = test_dataset[0]\n X = output[\"eeg\"]\n print(f\"X shape: {X.shape}\")\n \n test_loader = DataLoader(\n test_dataset,\n batch_size=CFG.batch_size,\n shuffle=False,\n num_workers=CFG.num_workers,\n pin_memory=True,\n drop_last=False,\n )\n\n model = EEGNet(\n kernels=CFG.kernels,\n in_channels=CFG.in_channels,\n fixed_kernel_size=CFG.fixed_kernel_size,\n num_classes=CFG.target_size,\n linear_layer_features=CFG.linear_layer_features,\n )\n\n for file_line in model_block['file_data']:\n koef = file_line['koef']\n for weight_model_file in glob(file_line['file_mask']):\n files.append(weight_model_file)\n checkpoint = torch.load(weight_model_file, map_location=device)\n model.load_state_dict(checkpoint[\"model\"])\n model.to(device)\n prediction_dict = inference_function(test_loader, model, device)\n predict = prediction_dict[\"predictions\"]\n predict *= koef\n koef_sum += koef\n koef_count += 1\n predictions.append(predict)\n torch.cuda.empty_cache()\n gc.collect()\n\npredictions = np.array(predictions)\nkoef_sum /= 0.00001\npredictions /= koef_sum\npredictions = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:14:29.870930Z","iopub.execute_input":"2024-04-06T21:14:29.871265Z","iopub.status.idle":"2024-04-06T21:14:29.899954Z","shell.execute_reply.started":"2024-04-06T21:14:29.871241Z","shell.execute_reply":"2024-04-06T21:14:29.899082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predss_1 = predictions\npredss_1","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:14:17.285046Z","iopub.execute_input":"2024-04-06T21:14:17.285372Z","iopub.status.idle":"2024-04-06T21:14:17.291567Z","shell.execute_reply.started":"2024-04-06T21:14:17.285347Z","shell.execute_reply":"2024-04-06T21:14:17.290860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import librosa\nimport os, random\nimport tensorflow\nimport tensorflow as tf\nimport albumentations as albu\nimport pandas as pd, numpy as np\nfrom scipy.signal import butter, lfilter\nimport tensorflow.keras.backend as K, gc\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate, LayerNormalization\n\nLOAD_BACKBONE_FROM = '../kaggle/input/efficientnetb-tf-keras/EfficientNetB2.h5'\nLOAD_MODELS_FROM = '../kaggle/input/features-head-starter-models'\nMODEL = {'K+E+KE': 52}\nfor DATA_TYPE in MODEL: pass\nTARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nFEATS2 = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\nFEAT2IDX = {x:y for x,y in zip(FEATS2,range(len(FEATS2)))}\nFEATS = [['Fp1','F7','T3','T5','O1'],\n ['Fp1','F3','C3','P3','O1'],\n ['Fp2','F8','T4','T6','O2'],\n ['Fp2','F4','C4','P4','O2']]\n \nclass DataGenerator():\n 'Generates data for Keras'\n def __init__(self, data, specs=None, eeg_specs=None, raw_eegs=None , augment=False, mode='train', data_type=DATA_TYPE): \n self.augment = augment\n self.mode = mode\n self.data_type = data_type\n self.data = self.build_data(data.copy())\n self.specs = specs\n self.eeg_specs = eeg_specs\n self.raw_eegs = raw_eegs\n self.on_epoch_end()\n \n def build_data(self,data):\n if self.data_type in ['K+E']:\n data_dup = pd.concat([data] * 2, ignore_index=True)\n data_dup.loc[:len(data),'data_type'] = 'K'\n data_dup.loc[len(data):,'data_type'] = 'E'\n data = data_dup\n elif self.data_type in ['K+E+KE']:\n data_trp = pd.concat([data] * 3, ignore_index=True)\n data_trp.loc[:len(data),'data_type'] = 'K'\n data_trp.loc[len(data):len(data)*2,'data_type'] = 'E'\n data_trp.loc[len(data)*2:,'data_type'] = 'KE'\n data = data_trp\n else:\n data['data_type'] = self.data_type\n return data\n \n def __len__(self):\n return self.data.shape[0]\n\n def __getitem__(self, index):\n X, y = self.data_generation(index)\n if self.augment: X = self.augmentation(X)\n return X, y\n \n def __call__(self):\n for i in range(self.__len__()):\n yield self.__getitem__(i)\n \n if i == self.__len__()-1:\n self.on_epoch_end()\n \n def on_epoch_end(self):\n if self.mode=='train': \n self.data = self.data.sample(frac=1).reset_index(drop=True)\n \n def data_generation(self, index):\n row = self.data.iloc[index]\n if row.data_type == 'KE':\n X,y = self.generate_all_specs(index)\n elif row.data_type in ['K','E']:\n X,y = self.generate_specs(index)\n elif row.data_type == 'R':\n X,y = self.generate_raw(index)\n elif row.data_type in ['ER','KR']:\n X1,y = self.generate_specs(index)\n X2,y = self.generate_raw(index)\n X = (X1,X2)\n elif row.data_type in ['KER']:\n X1,y = self.generate_all_specs(index)\n X2,y = self.generate_raw(index)\n X = (X1,X2)\n return X,y\n \n def generate_all_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n eeg = self.eeg_specs[row.eeg_id]\n spec = self.specs[row.spec_id]\n \n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in [0,2,1,3]] # to match kaggle with eeg\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n \n X[0_0+56:100+56,:256,0] = img[:,22:-22,0] # LL_k\n X[100+56:200+56,:256,0] = img[:,22:-22,2] # RL_k\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1] # LP_k\n X[100+56:200+56,:256,1] = img[:,22:-22,3] # RP_k\n X[0_0+56:100+56,:256,2] = img[:,22:-22,2] # RL_k\n X[100+56:200+56,:256,2] = img[:,22:-22,1] # LP_k\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0] # LL_k\n X[100+56:200+56,256:,0] = img[:,22:-22,2] # RL_k\n X[0_0+56:100+56,256:,1] = img[:,22:-22,1] # LP_k\n X[100+56:200+56,256:,1] = img[:,22:-22,3] # RP_K\n \n # EEG\n img = eeg\n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n X[200+56:300+56,:256,0] = img[:,22:-22,0] # LL_e\n X[300+56:400+56,:256,0] = img[:,22:-22,2] # RL_e\n X[200+56:300+56,:256,1] = img[:,22:-22,1] # LP_e\n X[300+56:400+56,:256,1] = img[:,22:-22,3] # RP_e\n X[200+56:300+56,:256,2] = img[:,22:-22,2] # RL_e\n X[300+56:400+56,:256,2] = img[:,22:-22,1] # LP_e\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0] # LL_e\n X[300+56:400+56,256:,0] = img[:,22:-22,2] # RL_e\n X[200+56:300+56,256:,1] = img[:,22:-22,1] # LP_e\n X[300+56:400+56,256:,1] = img[:,22:-22,3] # RP_e\n\n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n if row.data_type in ['E','ER']:\n img = self.eeg_specs[row.eeg_id]\n elif row.data_type in ['K','KR']:\n spec = self.specs[row.spec_id]\n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in [0,2,1,3]] # to match kaggle with eeg\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n \n X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n X[100+56:200+56,:256,0] = img[:,22:-22,2]\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n X[100+56:200+56,:256,1] = img[:,22:-22,3]\n X[0_0+56:100+56,:256,2] = img[:,22:-22,2]\n X[100+56:200+56,:256,2] = img[:,22:-22,1]\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n X[100+56:200+56,256:,0] = img[:,22:-22,1]\n X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n X[100+56:200+56,256:,1] = img[:,22:-22,3]\n \n X[200+56:300+56,:256,0] = img[:,22:-22,0]\n X[300+56:400+56,:256,0] = img[:,22:-22,1]\n X[200+56:300+56,:256,1] = img[:,22:-22,2]\n X[300+56:400+56,:256,1] = img[:,22:-22,3]\n X[200+56:300+56,:256,2] = img[:,22:-22,3]\n X[300+56:400+56,:256,2] = img[:,22:-22,2]\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0]\n X[300+56:400+56,256:,0] = img[:,22:-22,2]\n X[200+56:300+56,256:,1] = img[:,22:-22,1]\n X[300+56:400+56,256:,1] = img[:,22:-22,3]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_raw(self,index):\n if USE_PROCESSED and self.mode!='test':\n X = np.zeros((2_000,8),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n row = self.data.iloc[index]\n X = self.raw_eegs[row.eeg_id]\n y[:] = row[TARGETS]\n return X,y\n \n X = np.zeros((10_000,8),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n eeg = self.raw_eegs[row.eeg_id]\n \n # FEATURE ENGINEER\n X[:,0] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['T3']]\n X[:,1] = eeg[:,FEAT2IDX['T3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,2] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['C3']]\n X[:,3] = eeg[:,FEAT2IDX['C3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,4] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['C4']]\n X[:,5] = eeg[:,FEAT2IDX['C4']] - eeg[:,FEAT2IDX['O2']]\n \n X[:,6] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['T4']]\n X[:,7] = eeg[:,FEAT2IDX['T4']] - eeg[:,FEAT2IDX['O2']]\n \n # STANDARDIZE\n X = np.clip(X,-1024,1024)\n X = np.nan_to_num(X, nan=0) / 32.0\n \n # BUTTER LOW-PASS FILTER\n X = self.butter_lowpass_filter(X)\n # Downsample\n X = X[::5,:]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def butter_lowpass_filter(self, data, cutoff_freq=20, sampling_rate=200, order=4):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n b, a = butter(order, normal_cutoff, btype='low', analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n return filtered_data\n \n def resize(self, img,size):\n composition = albu.Compose([\n albu.Resize(size[0],size[1])\n ])\n return composition(image=img)['image']\n \n def augmentation(self, img):\n composition = albu.Compose([\n albu.HorizontalFlip(p=0.4)\n ])\n return composition(image=img)['image']\n\ndef spectrogram_from_eeg(parquet_path):\n \n # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n eeg = pd.read_parquet(parquet_path)\n middle = (len(eeg)-10_000)//2\n eeg = eeg.iloc[middle:middle+10_000]\n \n # VARIABLE TO HOLD SPECTROGRAM\n img = np.zeros((100,300,4),dtype='float32')\n\n for k in range(4):\n COLS = FEATS[k]\n \n for kk in range(4):\n # FILL NANS\n x1 = eeg[COLS[kk]].values\n x2 = eeg[COLS[kk+1]].values\n m = np.nanmean(x1)\n if np.isnan(x1).mean()<1: x1 = np.nan_to_num(x1,nan=m)\n else: x1[:] = 0\n m = np.nanmean(x2)\n if np.isnan(x2).mean()<1: x2 = np.nan_to_num(x2,nan=m)\n else: x2[:] = 0\n \n # COMPUTE PAIR DIFFERENCES\n x = x1 - x2\n\n # RAW SPECTROGRAM\n mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//300, \n n_fft=1024, n_mels=100, fmin=0, fmax=20, win_length=128)\n \n # LOG TRANSFORM\n width = (mel_spec.shape[1]//30)*30\n mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n img[:,:,k] += mel_spec_db\n \n # AVERAGE THE 4 MONTAGE DIFFERENCES\n img[:,:,k] /= 4.0\n \n return img\n\ndef eeg_from_parquet(parquet_path):\n\n eeg = pd.read_parquet(parquet_path, columns=FEATS2)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n data = np.zeros((10_000,len(FEATS2)))\n for j,col in enumerate(FEATS2):\n \n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n \n data[:,j] = x\n\n return data\n\ndef build_spec_model(hybrid=False): \n inp = tf.keras.layers.Input((512,512,3))\n base_model = load_model(f'{LOAD_BACKBONE_FROM}') \n x = base_model(inp)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n if not hybrid:\n x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n model = tf.keras.Model(inputs=inp, outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer=opt) \n return model\n\ndef dataset(data, mode='train', batch_size=32, data_type=DATA_TYPE, \n augment=False, specs=None, eeg_specs=None, raw_eegs=None):\n \n gen = DataGenerator(data,mode=mode, data_type=data_type, augment=augment,\n specs=specs, eeg_specs=eeg_specs, raw_eegs=raw_eegs)\n inp = tf.TensorSpec(shape=(512,512,3), dtype=tf.float32) \n output_signature = (inp,tf.TensorSpec(shape=(6,), dtype=tf.float32))\n dataset = tf.data.Dataset.from_generator(generator=gen, output_signature=output_signature).batch(\n batch_size)\n return dataset\n\ndef predict(models, params, fold, models_path=None):\n preds = []\n if models_path is None: models_path = LOAD_MODELS_FROM\n model = build_spec_model()\n for data_type in models:\n data = params['data']\n ver = models[data_type]\n ds = dataset(data_type=data_type, **params)\n model.load_weights(f'{models_path}/model_{data_type}_{ver}_{fold}.weights.h5')\n pred = model.predict(ds)\n if data_type in ['K+E+KE']:\n pred = (pred[:len(data)] + pred[len(data):len(data)*2] + pred[len(data)*2:])/3\n preds.append(pred)\n pred = np.mean(preds,axis=0)\n del model\n gc.collect()\n return pred","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:15:36.235814Z","iopub.execute_input":"2024-04-06T21:15:36.236192Z","iopub.status.idle":"2024-04-06T21:15:48.198750Z","shell.execute_reply.started":"2024-04-06T21:15:36.236163Z","shell.execute_reply":"2024-04-06T21:15:48.197913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms'\nfiles2 = os.listdir(PATH2)\nspectrograms2 = {}\nfor i,f in enumerate(files2):\n if i%100==0: print(i,', ',end='')\n tmp = pd.read_parquet(f'{PATH2}/{f}')\n name = int(f.split('.')[0])\n spectrograms2[name] = tmp.iloc[:,1:].values\n \n# RENAME FOR DATA GENERATOR\ntest = test.rename({'spectrogram_id':'spec_id'},axis=1)\n\n# READ ALL EEG SPECTROGRAMS\nPATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs'\nEEG_IDS2 = test.eeg_id.unique()\nall_eegs2 = {}\nfor i,eeg_id in enumerate(EEG_IDS2):\n \n # CREATE SPECTROGRAM FROM EEG PARQUET\n img = spectrogram_from_eeg(f'{PATH2}/{eeg_id}.parquet')\n all_eegs2[eeg_id] = img\n\n# READ ALL RAW EEG SIGNALS\nall_raw_eegs2 = {}\nfor i,eeg_id in enumerate(EEG_IDS2):\n \n # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n data = eeg_from_parquet(f'{PATH2}/{eeg_id}.parquet')\n all_raw_eegs2[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:16:00.356884Z","iopub.execute_input":"2024-04-06T21:16:00.357569Z","iopub.status.idle":"2024-04-06T21:16:12.547476Z","shell.execute_reply.started":"2024-04-06T21:16:00.357536Z","shell.execute_reply":"2024-04-06T21:16:12.546069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nparams = {'data':test,'mode':'test','specs':spectrograms2, 'eeg_specs':all_eegs2, 'raw_eegs':all_raw_eegs2}\n\nfor i in range(5):\n print(f'Fold {i+1}')\n pred = predict(MODEL,params,i)\n preds.append(pred)\n \npred = np.mean(preds,axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predss_2 = pred\npredss_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom IPython.display import display\n\nimport timm\nimport torch\nimport torch.nn as nn \nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\n\nfrom scipy import signal\n\nwarnings.filterwarnings('ignore', category=Warning)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:17:19.920705Z","iopub.execute_input":"2024-04-06T21:17:19.921369Z","iopub.status.idle":"2024-04-06T21:17:21.468924Z","shell.execute_reply.started":"2024-04-06T21:17:19.921337Z","shell.execute_reply":"2024-04-06T21:17:21.467871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n seed = 3131\n image_transform = transforms.Resize((512, 512))\n num_folds = 5\n dataset_wide_mean = -0.2972692229201065 #From Train notebook\n dataset_wide_std = 2.5997336315611026 #From Train notebook\n ownspec_mean = 7.29084372799223e-05 # From Train spectrograms notebook\n ownspec_std = 4.510082606216031 # From Train spectrograms notebook\n \ndef set_seed(seed):\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n \n torch.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n \nset_seed(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:17:40.439931Z","iopub.execute_input":"2024-04-06T21:17:40.440644Z","iopub.status.idle":"2024-04-06T21:17:40.449135Z","shell.execute_reply.started":"2024-04-06T21:17:40.440615Z","shell.execute_reply":"2024-04-06T21:17:40.448071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\n\nsubmission = submission.merge(test_df, on='eeg_id', how='left')\nsubmission['path_spec'] = submission['spectrogram_id'].apply(lambda x: f\"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/{x}.parquet\")\nsubmission['path_eeg'] = submission['eeg_id'].apply(lambda x: f\"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/{x}.parquet\")\n\ndisplay(submission)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:18:03.286071Z","iopub.execute_input":"2024-04-06T21:18:03.286473Z","iopub.status.idle":"2024-04-06T21:18:03.799306Z","shell.execute_reply.started":"2024-04-06T21:18:03.286443Z","shell.execute_reply":"2024-04-06T21:18:03.798369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\n\n# Load in original EfficientnetB0 model\nfor i in range(Config.num_folds):\n model_effnet_b0 = timm.create_model('efficientnet_b0', pretrained=False, num_classes=6, in_chans=1)\n model_effnet_b0.load_state_dict(torch.load(f'/kaggle/input/hms-train-efficientnetb0/efficientnet_b0_fold{i}.pth', map_location=torch.device('cpu')))\n models.append(model_effnet_b0)\n \nmodels_datawide = []\n# Load in hyperparameter optimized EfficientnetB1\nfor i in range(Config.num_folds):\n model_effnet_b1 = timm.create_model('efficientnet_b1', pretrained=False, num_classes=6, in_chans=1)\n model_effnet_b1.load_state_dict(torch.load(f'/kaggle/input/train/efficientnet_b1_fold{i}.pth', map_location=torch.device('cpu')))\n models_datawide.append(model_effnet_b1)\n \nmodels_ownspec = []\n# Load in EfficientnetB1 with new spectrograms\nfor i in range(Config.num_folds):\n model_effnet_b1 = timm.create_model('efficientnet_b1', pretrained=False, num_classes=6, in_chans=1)\n model_effnet_b1.load_state_dict(torch.load(f'/kaggle/input/efficientnet-b1-ownspectrograms/efficientnet_b1_fold{i}_datawide_CosineAnnealingLR_0.001_False.pth', map_location=torch.device('cpu')))\n models_ownspec.append(model_effnet_b1)\n \ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = []\n\ndef create_spectrogram(data):\n \"\"\"This function will create a spectrogram based on EEG-data\"\"\"\n nperseg = 150 # Length of each segment\n noverlap = 128 # Overlap between segments\n NFFT = max(256, 2 ** int(np.ceil(np.log2(nperseg))))\n\n # LL Spec = ( spec(Fp1 - F7) + spec(F7 - T3) + spec(T3 - T5) + spec(T5 - O1) )/4\n freqs, t,spectrum_LL1 = signal.spectrogram(data['Fp1']-data['F7'],nfft=NFFT,noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LL2 = signal.spectrogram(data['F7']-data['T3'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LL3 = signal.spectrogram(data['T3']-data['T5'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LL4 = signal.spectrogram(data['T5']-data['O1'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n\n LL = (spectrum_LL1+ spectrum_LL2 +spectrum_LL3 + spectrum_LL4)/4\n\n # LP Spec = ( spec(Fp1 - F3) + spec(F3 - C3) + spec(C3 - P3) + spec(P3 - O1) )/4\n freqs, t,spectrum_LP1 = signal.spectrogram(data['Fp1']-data['F3'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LP2 = signal.spectrogram(data['F3']-data['C3'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LP3 = signal.spectrogram(data['C3']-data['P3'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_LP4 = signal.spectrogram(data['P3']-data['O1'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n\n LP = (spectrum_LP1+ spectrum_LP2 +spectrum_LP3 + spectrum_LP4)/4\n\n # RP Spec = ( spec(Fp2 - F4) + spec(F4 - C4) + spec(C4 - P4) + spec(P4 - O2) )/4\n freqs, t,spectrum_RP1 = signal.spectrogram(data['Fp2']-data['F4'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RP2 = signal.spectrogram(data['F4']-data['C4'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RP3 = signal.spectrogram(data['C4']-data['P4'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RP4 = signal.spectrogram(data['P4']-data['O2'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n\n RP = (spectrum_RP1+ spectrum_RP2 +spectrum_RP3 + spectrum_RP4)/4\n\n\n # RL Spec = ( spec(Fp2 - F8) + spec(F8 - T4) + spec(T4 - T6) + spec(T6 - O2) )/4\n freqs, t,spectrum_RL1 = signal.spectrogram(data['Fp2']-data['F8'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RL2 = signal.spectrogram(data['F8']-data['T4'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RL3 = signal.spectrogram(data['T4']-data['T6'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n freqs, t,spectrum_RL4 = signal.spectrogram(data['T6']-data['O2'],nfft=NFFT, noverlap = noverlap,nperseg=nperseg)\n RL = (spectrum_RL1+ spectrum_RL2 +spectrum_RL3 + spectrum_RL4)/4\n spectogram = np.concatenate((LL, LP,RP,RL), axis=0)\n return spectogram\n\ndef preprocess_ownspec(path_to_parquet):\n \"\"\"The data will be processed from EEG to spectrogramdata\"\"\"\n data = pd.read_parquet(path_to_parquet)\n data = create_spectrogram(data)\n mask = np.isnan(data)\n data[mask] = -1\n data = np.clip(data, np.exp(-6), np.exp(10))\n data = np.log(data)\n \n return data \n\ndef preprocess(path_to_parquet):\n data = pd.read_parquet(path_to_parquet)\n data = data.fillna(-1).values[:, 1:].T\n data = np.clip(data, np.exp(-6), np.exp(10))\n data = np.log(data)\n \n return data\n\n\ndef normalize_datawide(data_point):\n \"\"\"The spectrogram data will be normalized data wide.\"\"\"\n eps = 1e-6\n\n data_point = (data_point - Config.dataset_wide_mean) / (Config.dataset_wide_std + eps)\n\n data_tensor = torch.unsqueeze(torch.Tensor(data_point), dim=0)\n data_point = Config.image_transform(data_tensor)\n\n return data_point\n\n\ndef normalize_datawide_ownspec(data):\n \"\"\"The new spectrogram data will be normalized data wide.\"\"\"\n eps = 1e-6\n \n data = (data - Config.ownspec_mean) / (Config.ownspec_std + eps)\n data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n data = Config.image_transform(data_tensor)\n \n return data\n\n\ndef normalize_instance_wise(data_point):\n \"\"\"The spectrogram data will be normalized instance wise.\"\"\"\n eps = 1e-6\n \n data_mean = data_point.mean(axis=(0, 1))\n data_std = data_point.std(axis=(0, 1))\n data_point = (data_point - data_mean) / (data_std + eps)\n \n data_tensor = torch.unsqueeze(torch.Tensor(data_point), dim=0)\n data_point = Config.image_transform(data_tensor)\n \n return data_point\n\n# Loop over samples\nfor index in submission.index:\n test_predictions_per_model = []\n \n preprocessed_data = preprocess(submission.iloc[index]['path_spec'])\n preprocessed_data_ownspec = preprocess_ownspec(submission.iloc[index]['path_eeg'])\n \n # Predict based on original EfficientnetB0 models. \n for i in range(len(models)):\n models[i].eval()\n \n current_parquet_data = normalize_instance_wise(preprocessed_data).unsqueeze(0)\n \n with torch.no_grad():\n model_output = models[i](current_parquet_data)\n current_model_prediction = F.softmax(model_output)[0].detach().cpu().numpy()\n \n test_predictions_per_model.append(current_model_prediction)\n \n # Predict based on hyperparameter optimized EffcientnetB1.\n for i in range(len(models_datawide)):\n models_datawide[i].eval()\n \n current_parquet_data = normalize_datawide(preprocessed_data).unsqueeze(0)\n \n with torch.no_grad():\n model_output = models_datawide[i](current_parquet_data)\n current_model_prediction = F.softmax(model_output)[0].detach().cpu().numpy()\n \n test_predictions_per_model.append(current_model_prediction)\n \n # Predict based on EfficientnetB1 model with new spectrograms.\n for i in range(len(models_ownspec)):\n models_ownspec[i].eval()\n \n current_parquet_data = normalize_datawide_ownspec(preprocessed_data_ownspec).unsqueeze(0)\n \n with torch.no_grad():\n model_output = models_ownspec[i](current_parquet_data)\n current_model_prediction = F.softmax(model_output)[0].detach().cpu().numpy()\n \n test_predictions_per_model.append(current_model_prediction)\n \n # The mean of all models is taken.\n ensemble_prediction = np.mean(test_predictions_per_model,axis=0)\n \n test_predictions.append(ensemble_prediction)\n\ntest_predictions = np.array(test_predictions)\n\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predss_3 = test_predictions\npredss_3","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\nlabels=['seizure','lpd','gpd','lrda','grda','other']\nfor i in range(len(labels)):\n submission[f'{labels[i]}_vote']=(predss_1[:,i] * 0.2 + predss_2[:, i] * 0.4 + predss_3[:, i] * 0.4)\nsubmission.to_csv(\"submission.csv\",index=None)\ndisplay(submission.head())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.iloc[:,-6:].sum(axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"model - 4","metadata":{}},{"cell_type":"code","source":"import librosa\nimport os, random\nimport tensorflow\nimport tensorflow as tf\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport pandas as pd, numpy as np\nfrom scipy.signal import butter, lfilter\nimport tensorflow.keras.backend as K, gc\nfrom tensorflow.keras.models import load_model\nfrom sklearn.model_selection import KFold, GroupKFold\n\nLOAD_BACKBONE_FROM = '/kaggle/input/efficientnetb-tf-keras/EfficientNetB2.h5'\nLOAD_MODELS_FROM = '/kaggle/input/features-head-starter-models'\nHMS_PATH = '/kaggle/input/hms-harmful-brain-activity-classification'\nMODEL = {'KER': 64} # Setup the model for Training\nfor DATA_TYPE in MODEL: pass # K|E|R|R2|KE|KR|ER|KER|K+E|K+E+KE\nUSE_PROCESSED = True # Use processed downsampled Raw EEG \nKNOWLEDGE_DISTILLATION = True # Use distilled dataset for training\nTEST_MODE = False # USE 500 samples for quick testing\nTEST_ENSEMBLE = False # Local Ensemble CV Testing\nSUBMISSION = True\n\n# Setup for ensemble\nENSEMBLE = False\nMODELS = {\n 'K' : 43, # 'LB':0.41 Kaggle's spectrogram model version\n 'E' : 42, # 'LB':0.39 EEG's spectrogram model version\n 'R' : 60, # 'LB':0.4 EEG's Raw wavenet model version, trained on single GPU\n 'R2' : 61, # 'LB':0.46 EEG's Raw ResNet1D model version, trained on single GPU\n 'KE' : 58, # 'LB':0.34 Kaggle's and EEG's spectrogram model version\n 'KR' : 48, # 'LB':0.39 Kaggle's spectrogram and Raw model version\n 'ER' : 49, # 'LB':0.38 EEG's spectrogram and Raw model version\n 'KER' : 64, # 'LB':0.33 EEG's, Kaggle's spectrograms and Raw model version\n 'K+E' : 51, # 'LB':0.37 Data spectrogram augmentation model version\n 'K+E+KE': 59, # 'LB':0.34 Data spectrogram augmentation model version\n }\n\nnp.random.seed(42)\nrandom.seed(42)\ntf.random.set_seed(42)\n\n# USE SINGLE GPU, MULTIPLE GPUS \ngpus = tf.config.list_physical_devices('GPU')\n# WE USE MIXED PRECISION\ntf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\nif len(gpus)>1:\n strategy = tf.distribute.MirroredStrategy()\n print(f'Using {len(gpus)} GPUs')\nelse:\n strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n print(f'Using {len(gpus)} GPU')","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:25:31.383760Z","iopub.execute_input":"2024-04-06T21:25:31.384485Z","iopub.status.idle":"2024-04-06T21:25:45.691717Z","shell.execute_reply.started":"2024-04-06T21:25:31.384450Z","shell.execute_reply":"2024-04-06T21:25:45.690749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nFEATS2 = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\n\ndef eeg_from_parquet(parquet_path):\n\n eeg = pd.read_parquet(parquet_path, columns=FEATS2)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n data = np.zeros((10_000,len(FEATS2)))\n for j,col in enumerate(FEATS2):\n \n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n \n data[:,j] = x\n\n return data\n\ndef add_kl(data):\n labels = data[TARGETS].values + 1e-5\n data['kl'] = tf.keras.losses.KLDivergence(reduction='none')(\n np.array([[1/6]*6]*len(data)),labels)\n return data\n \nif not SUBMISSION:\n train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n META = ['spectrogram_id','spectrogram_label_offset_seconds','patient_id','expert_consensus']\n train = train.groupby('eeg_id')[META+TARGETS\n ].agg({**{m:'first' for m in META},**{t:'sum' for t in TARGETS}}).reset_index() \n train[TARGETS] = train[TARGETS]/train[TARGETS].values.sum(axis=1,keepdims=True)\n train.columns = ['eeg_id','spec_id','offset','patient_id','target'] + TARGETS\n train = add_kl(train)\n pred = pd.read_csv('/kaggle/input/knowledge-distillation/dataset_KD.csv')\n train_KD = train.copy()\n train_KD[TARGETS] = pred[TARGETS]\n print(train.head(1).to_string())\n print(train_KD.head(1).to_string())","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:25:57.626668Z","iopub.execute_input":"2024-04-06T21:25:57.627042Z","iopub.status.idle":"2024-04-06T21:25:57.640460Z","shell.execute_reply.started":"2024-04-06T21:25:57.627016Z","shell.execute_reply":"2024-04-06T21:25:57.639426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not SUBMISSION:\n # FOR TESTING SET TEST_MODE TO TRUE\n if TEST_MODE:\n USE_PROCESSED = False\n train = train.sample(500,random_state=42).reset_index(drop=True)\n spectrograms = {}\n for i,e in enumerate(train.spec_id.values):\n if i%100==0: print(i,', ',end='')\n x = pd.read_parquet(f'{HMS_PATH}/train_spectrograms/{e}.parquet')\n spectrograms[e] = x.values\n all_eegs = {}\n for i,e in enumerate(train.eeg_id.values):\n if i%100==0: print(i,', ',end='')\n x = np.load(f'/kaggle/input/eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n all_eegs[e] = x\n all_raw_eegs = {}\n for i,e in enumerate(train.eeg_id.values):\n if i%100==0: print(i,', ',end='')\n x = eeg_from_parquet(f'{HMS_PATH}/train_eegs/{e}.parquet') \n all_raw_eegs[e] = x\n else:\n spectrograms = None\n all_eegs = None\n all_raw_eegs = None\n if TEST_ENSEMBLE or DATA_TYPE in ['K','KE','K+E','K+E+KE','KR','KER']:\n spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()\n if TEST_ENSEMBLE or DATA_TYPE in ['E','KE','K+E','K+E+KE','ER','KER']:\n all_eegs = np.load('/kaggle/input/eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()\n if TEST_ENSEMBLE or DATA_TYPE in ['R','R2','KR','ER','KER']:\n if USE_PROCESSED:\n all_raw_eegs = np.load('/kaggle/input/hms-eeg/eegs_processed.npy',allow_pickle=True).item()\n else:\n all_raw_eegs = np.load('/kaggle/input/hms-eeg/eegs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:26:14.187090Z","iopub.execute_input":"2024-04-06T21:26:14.187443Z","iopub.status.idle":"2024-04-06T21:26:14.199121Z","shell.execute_reply.started":"2024-04-06T21:26:14.187416Z","shell.execute_reply":"2024-04-06T21:26:14.198269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATS2 = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\nFEAT2IDX = {x:y for x,y in zip(FEATS2,range(len(FEATS2)))}\nFEATS = [['Fp1','F7','T3','T5','O1'],\n ['Fp1','F3','C3','P3','O1'],\n ['Fp2','F8','T4','T6','O2'],\n ['Fp2','F4','C4','P4','O2']]\n \nclass DataGenerator():\n 'Generates data for Keras'\n def __init__(self, data, specs=None, eeg_specs=None, raw_eegs=None , augment=False, mode='train', data_type=DATA_TYPE): \n self.augment = augment\n self.mode = mode\n self.data_type = data_type\n self.data = self.build_data(data.copy())\n self.specs = specs\n self.eeg_specs = eeg_specs\n self.raw_eegs = raw_eegs\n self.on_epoch_end()\n \n def build_data(self,data):\n if self.data_type in ['K+E']:\n data_dup = pd.concat([data] * 2, ignore_index=True)\n data_dup.loc[:len(data),'data_type'] = 'K'\n data_dup.loc[len(data):,'data_type'] = 'E'\n data = data_dup\n elif self.data_type in ['K+E+KE']:\n data_trp = pd.concat([data] * 3, ignore_index=True)\n data_trp.loc[:len(data),'data_type'] = 'K'\n data_trp.loc[len(data):len(data)*2,'data_type'] = 'E'\n data_trp.loc[len(data)*2:,'data_type'] = 'KE'\n data = data_trp\n else:\n data['data_type'] = self.data_type\n return data\n \n def __len__(self):\n return self.data.shape[0]\n\n def __getitem__(self, index):\n X, y = self.data_generation(index)\n if self.augment: X = self.augmentation(X)\n return X, y\n \n def __call__(self):\n for i in range(self.__len__()):\n yield self.__getitem__(i)\n \n if i == self.__len__()-1:\n self.on_epoch_end()\n \n def on_epoch_end(self):\n if self.mode=='train': \n self.data = self.data.sample(frac=1).reset_index(drop=True)\n \n def data_generation(self, index):\n row = self.data.iloc[index]\n if row.data_type == 'KE':\n X,y = self.generate_all_specs(index)\n elif row.data_type in ['K','E']:\n X,y = self.generate_specs(index)\n elif row.data_type in ['R','R2']:\n X,y = self.generate_raw(index)\n elif row.data_type in ['ER','KR']:\n X1,y = self.generate_specs(index)\n X2,y = self.generate_raw(index)\n X = (X1,X2)\n elif row.data_type in ['KER']:\n X1,y = self.generate_all_specs(index)\n X2,y = self.generate_raw(index)\n X = (X1,X2)\n return X,y\n \n def generate_all_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n eeg = self.eeg_specs[row.eeg_id]\n spec = self.specs[row.spec_id]\n \n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in [0,2,1,3]] # to match kaggle with eeg\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n \n X[0_0+56:100+56,:256,0] = img[:,22:-22,0] # LL_k\n X[100+56:200+56,:256,0] = img[:,22:-22,2] # RL_k\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1] # LP_k\n X[100+56:200+56,:256,1] = img[:,22:-22,3] # RP_k\n X[0_0+56:100+56,:256,2] = img[:,22:-22,2] # RL_k\n X[100+56:200+56,:256,2] = img[:,22:-22,1] # LP_k\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0] # LL_k\n X[100+56:200+56,256:,0] = img[:,22:-22,2] # RL_k\n X[0_0+56:100+56,256:,1] = img[:,22:-22,1] # LP_k\n X[100+56:200+56,256:,1] = img[:,22:-22,3] # RP_K\n \n # EEG\n img = eeg\n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n X[200+56:300+56,:256,0] = img[:,22:-22,0] # LL_e\n X[300+56:400+56,:256,0] = img[:,22:-22,2] # RL_e\n X[200+56:300+56,:256,1] = img[:,22:-22,1] # LP_e\n X[300+56:400+56,:256,1] = img[:,22:-22,3] # RP_e\n X[200+56:300+56,:256,2] = img[:,22:-22,2] # RL_e\n X[300+56:400+56,:256,2] = img[:,22:-22,1] # LP_e\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0] # LL_e\n X[300+56:400+56,256:,0] = img[:,22:-22,2] # RL_e\n X[200+56:300+56,256:,1] = img[:,22:-22,1] # LP_e\n X[300+56:400+56,256:,1] = img[:,22:-22,3] # RP_e\n\n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n if row.data_type in ['E','ER']:\n img = self.eeg_specs[row.eeg_id]\n elif row.data_type in ['K','KR']:\n spec = self.specs[row.spec_id]\n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in [0,2,1,3]] # to match kaggle with eeg\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n \n X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n X[100+56:200+56,:256,0] = img[:,22:-22,2]\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n X[100+56:200+56,:256,1] = img[:,22:-22,3]\n X[0_0+56:100+56,:256,2] = img[:,22:-22,2]\n X[100+56:200+56,:256,2] = img[:,22:-22,1]\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n X[100+56:200+56,256:,0] = img[:,22:-22,1]\n X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n X[100+56:200+56,256:,1] = img[:,22:-22,3]\n \n X[200+56:300+56,:256,0] = img[:,22:-22,0]\n X[300+56:400+56,:256,0] = img[:,22:-22,1]\n X[200+56:300+56,:256,1] = img[:,22:-22,2]\n X[300+56:400+56,:256,1] = img[:,22:-22,3]\n X[200+56:300+56,:256,2] = img[:,22:-22,3]\n X[300+56:400+56,:256,2] = img[:,22:-22,2]\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0]\n X[300+56:400+56,256:,0] = img[:,22:-22,2]\n X[200+56:300+56,256:,1] = img[:,22:-22,1]\n X[300+56:400+56,256:,1] = img[:,22:-22,3]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_raw(self,index):\n if USE_PROCESSED and self.mode!='test':\n X = np.zeros((2_000,8),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n row = self.data.iloc[index]\n X = self.raw_eegs[row.eeg_id]\n y[:] = row[TARGETS]\n return X,y\n \n X = np.zeros((10_000,8),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n eeg = self.raw_eegs[row.eeg_id]\n \n # FEATURE ENGINEER\n X[:,0] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['T3']]\n X[:,1] = eeg[:,FEAT2IDX['T3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,2] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['C3']]\n X[:,3] = eeg[:,FEAT2IDX['C3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,4] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['C4']]\n X[:,5] = eeg[:,FEAT2IDX['C4']] - eeg[:,FEAT2IDX['O2']]\n \n X[:,6] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['T4']]\n X[:,7] = eeg[:,FEAT2IDX['T4']] - eeg[:,FEAT2IDX['O2']]\n \n # STANDARDIZE\n X = np.clip(X,-1024,1024)\n X = np.nan_to_num(X, nan=0) / 32.0\n \n # BUTTER LOW-PASS FILTER\n X = self.butter_lowpass_filter(X)\n # Downsample\n X = X[::5,:]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def butter_lowpass_filter(self, data, cutoff_freq=20, sampling_rate=200, order=4):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n b, a = butter(order, normal_cutoff, btype='low', analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n return filtered_data\n \n def resize(self, img,size):\n composition = albu.Compose([\n albu.Resize(size[0],size[1])\n ])\n return composition(image=img)['image']\n \n def augmentation(self, img):\n composition = albu.Compose([\n albu.HorizontalFlip(p=0.4)\n ])\n return composition(image=img)['image']\n\ndef spectrogram_from_eeg(parquet_path):\n \n # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n eeg = pd.read_parquet(parquet_path)\n middle = (len(eeg)-10_000)//2\n eeg = eeg.iloc[middle:middle+10_000]\n \n # VARIABLE TO HOLD SPECTROGRAM\n img = np.zeros((100,300,4),dtype='float32')\n\n for k in range(4):\n COLS = FEATS[k]\n \n for kk in range(4):\n # FILL NANS\n x1 = eeg[COLS[kk]].values\n x2 = eeg[COLS[kk+1]].values\n m = np.nanmean(x1)\n if np.isnan(x1).mean()<1: x1 = np.nan_to_num(x1,nan=m)\n else: x1[:] = 0\n m = np.nanmean(x2)\n if np.isnan(x2).mean()<1: x2 = np.nan_to_num(x2,nan=m)\n else: x2[:] = 0\n \n # COMPUTE PAIR DIFFERENCES\n x = x1 - x2\n\n # RAW SPECTROGRAM\n mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//300, \n n_fft=1024, n_mels=100, fmin=0, fmax=20, win_length=128)\n \n # LOG TRANSFORM\n width = (mel_spec.shape[1]//30)*30\n mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n img[:,:,k] += mel_spec_db\n \n # AVERAGE THE 4 MONTAGE DIFFERENCES\n img[:,:,k] /= 4.0\n \n return img\n\ndef eeg_from_parquet(parquet_path):\n\n eeg = pd.read_parquet(parquet_path, columns=FEATS2)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n data = np.zeros((10_000,len(FEATS2)))\n for j,col in enumerate(FEATS2):\n \n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n \n data[:,j] = x\n\n return data","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:27:01.112512Z","iopub.execute_input":"2024-04-06T21:27:01.112846Z","iopub.status.idle":"2024-04-06T21:27:01.183848Z","shell.execute_reply.started":"2024-04-06T21:27:01.112820Z","shell.execute_reply":"2024-04-06T21:27:01.183021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMISSION: \n params = {'specs':spectrograms, 'eeg_specs':all_eegs, 'raw_eegs':all_raw_eegs}\n gen = DataGenerator(train, augment=False, **params)\n for x,y in gen:\n break\n \n if DATA_TYPE in ['E','K','KE','K+E','K+E+KE','KR','ER','KER']:\n x1 = x[0] if DATA_TYPE in ['KR','ER','KER'] else x\n plt.imshow(x1[:,:,0])\n plt.title(f'Target = {y.round(1)}',size=12)\n plt.yticks([])\n plt.ylabel('Frequencies (Hz)',size=12)\n plt.xlabel('Time (sec)',size=12)\n \n if DATA_TYPE in ['R','R2','KR','ER','KER']:\n x1 = x[1] if DATA_TYPE in ['KR','ER','KER'] else x\n plt.figure(figsize=(20,4))\n offset = 0\n for j in range(x1.shape[-1]):\n if j!=0: offset -= x1[:,j].min()\n plt.plot(range(2_000),x1[:,j]+offset,label=f'feature {j+1}')\n offset += x1[:,j].max()\n plt.legend()\n \n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:27:21.171445Z","iopub.execute_input":"2024-04-06T21:27:21.172304Z","iopub.status.idle":"2024-04-06T21:27:21.181319Z","shell.execute_reply.started":"2024-04-06T21:27:21.172253Z","shell.execute_reply":"2024-04-06T21:27:21.180405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMISSION:\n\n def lrfn(epoch):\n e3 = 1e-3 if DATA_TYPE in ['R'] else 1e-4\n return [1e-3,1e-3,e3,1e-4,1e-5][epoch]\n\n LR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n \n def lrfn2(epoch):\n return [1e-5,1e-5,1e-6][epoch]\n\n LR2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:27:38.977473Z","iopub.execute_input":"2024-04-06T21:27:38.977823Z","iopub.status.idle":"2024-04-06T21:27:38.983898Z","shell.execute_reply.started":"2024-04-06T21:27:38.977796Z","shell.execute_reply":"2024-04-06T21:27:38.982818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate\nfrom tensorflow.keras.layers import BatchNormalization, Bidirectional,GRU, MaxPooling1D, ReLU\n\ndef build_model(data_type=DATA_TYPE):\n K.clear_session()\n with strategy.scope():\n if data_type in ['R']:\n model = build_wave_model()\n if data_type in ['R2']:\n model = build_Resnet1D_GRU_model()\n elif data_type in ['K','E','KE','K+E','K+E+KE']:\n model = build_spec_model()\n elif data_type in ['KR','ER','KER']:\n model = build_hybrid_model()\n return model\n\ndef build_spec_model(hybrid=False): \n inp = tf.keras.layers.Input((512,512,3))\n base_model = load_model(f'{LOAD_BACKBONE_FROM}') \n x = base_model(inp)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n if not hybrid:\n x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n model = tf.keras.Model(inputs=inp, outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer=opt) \n return model\n\ndef wave_block(x, filters, kernel_size, n):\n dilation_rates = [2**i for i in range(n)]\n x = Conv1D(filters = filters,\n kernel_size = 1,\n padding = 'same')(x)\n res_x = x\n for dilation_rate in dilation_rates:\n tanh_out = Conv1D(filters = filters,\n kernel_size = kernel_size,\n padding = 'same', \n activation = 'tanh', \n dilation_rate = dilation_rate)(x)\n sigm_out = Conv1D(filters = filters,\n kernel_size = kernel_size,\n padding = 'same',\n activation = 'sigmoid', \n dilation_rate = dilation_rate)(x)\n x = Multiply()([tanh_out, sigm_out])\n x = Conv1D(filters = filters,\n kernel_size = 1,\n padding = 'same')(x)\n res_x = Add()([res_x, x])\n return res_x\n\ndef build_wave_model(hybrid=False):\n \n # INPUT \n inp = tf.keras.Input(shape=(2_000,8))\n \n ############\n # FEATURE EXTRACTION SUB MODEL\n inp2 = tf.keras.Input(shape=(2_000,1))\n x = wave_block(inp2, 8, 4, 6)\n x = wave_block(x, 16, 4, 6)\n x = wave_block(x, 32, 4, 6)\n x = wave_block(x, 64, 4, 6)\n model2 = tf.keras.Model(inputs=inp2, outputs=x)\n ###########\n \n # LEFT TEMPORAL CHAIN\n x1 = model2(inp[:,:,0:1])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,1:2])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z1 = tf.keras.layers.Average()([x1,x2])\n \n # LEFT PARASAGITTAL CHAIN\n x1 = model2(inp[:,:,2:3])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,3:4])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z2 = tf.keras.layers.Average()([x1,x2])\n \n # RIGHT PARASAGITTAL CHAIN\n x1 = model2(inp[:,:,4:5])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,5:6])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z3 = tf.keras.layers.Average()([x1,x2])\n \n # RIGHT TEMPORAL CHAIN\n x1 = model2(inp[:,:,6:7])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,7:8])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z4 = tf.keras.layers.Average()([x1,x2])\n \n # COMBINE CHAINS\n y = tf.keras.layers.Concatenate()([z1,z2,z3,z4])\n if not hybrid:\n y = tf.keras.layers.Dense(64, activation='relu')(y)\n y = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(y)\n \n # COMPILE MODEL\n model = tf.keras.Model(inputs=inp, outputs=y)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer = opt)\n \n return model\n\ndef build_hybrid_model():\n model_spec = build_spec_model(True)\n model_wave = build_wave_model(True)\n inputs = [model_spec.input, model_wave.input]\n x = [model_spec.output, model_wave.output]\n x = tf.keras.layers.Concatenate()(x)\n x = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n \n # COMPILE MODEL\n model = tf.keras.Model(inputs=inputs, outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer = opt)\n \n return model\n\ndef build_Resnet1D_GRU_model():\n inp = tf.keras.Input(shape=(2_000,8))\n x = [Conv1D(filters=24, kernel_size=kernel, padding='same', \n use_bias=False)(inp) for kernel in [3, 5, 7, 9, 11]]\n x = Concatenate()(x)\n x = BatchNormalization()(x)\n x = ReLU()(x)\n x = Conv1D(filters=24, kernel_size=5, strides=2, padding='same', use_bias=False)(x)\n \n for i in range(9): \n x_res = x\n x = BatchNormalization()(x)\n x = ReLU()(x)\n x = Conv1D(filters=24, kernel_size=5, strides=1, padding='same', use_bias=False)(x)\n x = BatchNormalization()(x)\n x = ReLU()(x)\n x = Conv1D(filters=24, kernel_size=5, strides=1, padding='same', use_bias=False)(x)\n if i%2:\n x = MaxPooling1D(pool_size=2, strides=2, padding='same')(x)\n x_res = MaxPooling1D(pool_size=2, strides=2, padding='same')(x_res)\n x += x_res\n \n x = BatchNormalization()(x)\n x = ReLU()(x)\n x = Bidirectional(GRU(128))(x)\n x = Dense(6,activation='softmax', dtype='float32')(x)\n \n # COMPILE MODEL\n model = tf.keras.Model(inputs=inp, outputs=x)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer = opt)\n \n return model\n\ndef score(y_true, y_pred):\n kl = tf.keras.metrics.KLDivergence()\n return kl(y_true, y_pred)\n\ndef plot_hist(hist):\n metrics = ['loss']\n for i,metric in enumerate(metrics):\n plt.figure(figsize=(10,4))\n plt.subplot(1,2,i+1)\n plt.plot(hist[metric])\n plt.plot(hist[f'val_{metric}'])\n plt.title(f'{metric}',size=12)\n plt.ylabel(f'{metric}',size=12)\n plt.xlabel('epoch',size=12)\n plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n plt.show()\n \ndef dataset(data, mode='train', batch_size=8, data_type=DATA_TYPE, \n augment=False, specs=None, eeg_specs=None, raw_eegs=None):\n \n BATCH_SIZE_PER_REPLICA = batch_size\n BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n gen = DataGenerator(data,mode=mode, data_type=data_type, augment=augment,\n specs=specs, eeg_specs=eeg_specs, raw_eegs=raw_eegs)\n if data_type in ['K','E','KE','K+E','K+E+KE']: \n inp = tf.TensorSpec(shape=(512,512,3), dtype=tf.float32)\n elif data_type in ['KR','ER','KER']:\n inp = (tf.TensorSpec(shape=(512,512,3), dtype=tf.float32),tf.TensorSpec(shape=(2000,8), dtype=tf.float32))\n elif data_type in ['R','R2']:\n inp = tf.TensorSpec(shape=(2000,8), dtype=tf.float32)\n \n output_signature = (inp,tf.TensorSpec(shape=(6,), dtype=tf.float32))\n dataset = tf.data.Dataset.from_generator(generator=gen, output_signature=output_signature).batch(\n BATCH_SIZE)\n return dataset\n\ndef reset_seed(seed):\n np.random.seed(seed)\n random.seed(seed)\n tf.random.set_seed(seed)\n \ndef get_size(x):\n if DATA_TYPE in ['K+E']:\n size = 2*len(x)\n elif DATA_TYPE in ['K+E+KE']:\n size = 3*len(x)\n else:\n size = len(x)\n return size\n\ndef predict(models, params, fold, models_path=None):\n preds = []\n if models_path is None: models_path = LOAD_MODELS_FROM\n model_wave = build_wave_model()\n model_spec = build_spec_model()\n model_hybrid = build_hybrid_model()\n model_resnet = build_Resnet1D_GRU_model()\n for data_type in models:\n data = params['data']\n ver = models[data_type]\n ds = dataset(data_type=data_type, **params)\n if data_type in ['R']:\n model = model_wave\n if data_type in ['R2']:\n model = model_resnet\n elif data_type in ['K','E','KE','K+E','K+E+KE']:\n model = model_spec\n elif data_type in ['KR','ER','KER']:\n model = model_hybrid\n model.load_weights(f'{models_path}/model_{data_type}_{ver}_{fold}.weights.h5')\n pred = model.predict(ds)\n if data_type in ['K+E']:\n pred = (pred[:len(data)] + pred[len(data):])/2\n if data_type in ['K+E+KE']:\n pred = (pred[:len(data)] + pred[len(data):len(data)*2] + pred[len(data)*2:])/3\n preds.append(pred)\n pred = np.mean(preds,axis=0)\n del model_wave, model_spec, model_hybrid\n gc.collect()\n return pred","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:28:21.394575Z","iopub.execute_input":"2024-04-06T21:28:21.394899Z","iopub.status.idle":"2024-04-06T21:28:21.444238Z","shell.execute_reply.started":"2024-04-06T21:28:21.394875Z","shell.execute_reply":"2024-04-06T21:28:21.443521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMISSION and not TEST_ENSEMBLE:\n reset_seed(42)\n all_oof = []\n all_true = []\n val_sizes = []\n losses = []\n val_losses = []\n total_hist = {}\n\n gkf = GroupKFold(n_splits=5)\n for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)): \n print('#'*25)\n print(f'### Fold {i+1}')\n \n params = {'specs':spectrograms, 'eeg_specs':all_eegs, 'raw_eegs':all_raw_eegs}\n data = train_KD.iloc[train_index] if KNOWLEDGE_DISTILLATION else train.iloc[train_index]\n val = train.iloc[valid_index]\n train_dataset = dataset(data, **params)\n val_dataset = dataset(val,mode='valid', **params)\n data = data[data['kl']<5.5]\n train_dataset2 = dataset(data, **params)\n train_size = get_size(train_index)\n valid_size = get_size(valid_index)\n print(f'### train size {train_size}, valid size {valid_size}')\n print('#'*25)\n model = build_model()\n hist = model.fit(train_dataset, validation_data = val_dataset, \n epochs=5, callbacks=[LR])\n train_size = get_size(data)\n print(f'### seconds stage train size {train_size}, valid size {valid_size}')\n print('#'*25)\n hist2 = model.fit(train_dataset2, validation_data = val_dataset, \n epochs=3, callbacks=[LR2])\n losses.append(hist.history['loss']+hist2.history['loss'])\n val_losses.append(hist.history['val_loss']+hist2.history['val_loss'])\n with strategy.scope():\n model.save_weights(f'model_{DATA_TYPE}_{MODEL[DATA_TYPE]}_{i}.weights.h5')\n params = {'mode':'valid','data':val,'specs':spectrograms, 'eeg_specs':all_eegs, 'raw_eegs':all_raw_eegs}\n oof = predict(MODEL, params, i, models_path='./')\n all_oof.append(oof)\n all_true.append(train.iloc[valid_index][TARGETS].values)\n val_sizes.append(len(valid_index))\n del model, oof\n gc.collect()\n \n total_hist['loss'] = np.mean(losses,axis=0)\n total_hist['val_loss'] = np.mean(val_losses,axis=0)\n all_oof = np.concatenate(all_oof)\n all_true = np.concatenate(all_true)\n plot_hist(total_hist)\n print(f'CV KL SCORE: {score(all_true,all_oof):.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:28:52.619862Z","iopub.execute_input":"2024-04-06T21:28:52.620220Z","iopub.status.idle":"2024-04-06T21:28:52.633447Z","shell.execute_reply.started":"2024-04-06T21:28:52.620193Z","shell.execute_reply":"2024-04-06T21:28:52.632555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMISSION and TEST_ENSEMBLE:\n MODELS = {\n 'K' : 43, # 'LB':0.41 Kaggle's spectrogram model version\n 'E' : 42, # 'LB':0.39 EEG's spectrogram model version\n 'R' : 60, # 'LB':0.4 EEG's Raw wavenet model version, trained on single GPU\n 'R2' : 61, # 'LB':0.46 EEG's Raw wavenet model version, trained on single GPU\n 'KE' : 58, # 'LB':0.34 Kaggle's and EEG's spectrogram model version\n 'KR' : 48, # 'LB':0.39 Kaggle's spectrogram and Raw model version\n 'ER' : 49, # 'LB':0.38 EEG's spectrogram and Raw model version\n 'KER' : 64, # 'LB':0.33 EEG's, Kaggle's spectrograms and Raw model version\n 'K+E' : 51, # 'LB':0.37 Data spectrogram augmentation model version\n 'K+E+KE': 59, # 'LB':0.34 Data spectrogram augmentation model version\n }\n reset_seed(42)\n gkf = GroupKFold(n_splits=5)\n all_oof = []\n all_true = []\n for i, (_, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)): \n print(f'Fold {i+1}')\n val_data = train.iloc[valid_index].copy()\n params = {'mode':'valid','data':val_data,'specs':spectrograms, 'eeg_specs':all_eegs, 'raw_eegs':all_raw_eegs}\n oof = predict(MODELS,params,i)\n all_oof.append(oof)\n all_true.append(val_data[TARGETS].values)\n \n all_oof = np.concatenate(all_oof)\n all_true = np.concatenate(all_true)\n print(f'CV KL SCORE: {score(all_true,all_oof):.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:29:07.619771Z","iopub.execute_input":"2024-04-06T21:29:07.620468Z","iopub.status.idle":"2024-04-06T21:29:07.628652Z","shell.execute_reply.started":"2024-04-06T21:29:07.620437Z","shell.execute_reply":"2024-04-06T21:29:07.627682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMISSION:\n test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\n print('Test shape',test.shape)\n test.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:29:16.774221Z","iopub.execute_input":"2024-04-06T21:29:16.774912Z","iopub.status.idle":"2024-04-06T21:29:16.787800Z","shell.execute_reply.started":"2024-04-06T21:29:16.774885Z","shell.execute_reply":"2024-04-06T21:29:16.786762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nif SUBMISSION:\n PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms'\n files2 = os.listdir(PATH2)\n print(f'There are {len(files2)} test spectrogram parquets')\n \n spectrograms2 = {}\n for i,f in enumerate(files2):\n if i%100==0: print(i,', ',end='')\n tmp = pd.read_parquet(f'{PATH2}/{f}')\n name = int(f.split('.')[0])\n spectrograms2[name] = tmp.iloc[:,1:].values\n \n # RENAME FOR DATA GENERATOR\n test = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:29:33.331649Z","iopub.execute_input":"2024-04-06T21:29:33.332352Z","iopub.status.idle":"2024-04-06T21:29:33.538456Z","shell.execute_reply.started":"2024-04-06T21:29:33.332325Z","shell.execute_reply":"2024-04-06T21:29:33.537495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nif SUBMISSION:\n PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs'\n DISPLAY = 0\n EEG_IDS2 = test.eeg_id.unique()\n all_eegs2 = {}\n\n print('Converting Test EEG to Spectrograms...'); print()\n for i,eeg_id in enumerate(EEG_IDS2):\n \n # CREATE SPECTROGRAM FROM EEG PARQUET\n img = spectrogram_from_eeg(f'{PATH2}/{eeg_id}.parquet')\n all_eegs2[eeg_id] = img","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:29:51.494821Z","iopub.execute_input":"2024-04-06T21:29:51.495825Z","iopub.status.idle":"2024-04-06T21:30:01.271289Z","shell.execute_reply.started":"2024-04-06T21:29:51.495784Z","shell.execute_reply":"2024-04-06T21:30:01.269882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL RAW EEG SIGNALS\nif SUBMISSION :\n all_raw_eegs2 = {}\n EEG_IDS2 = test.eeg_id.unique()\n PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs'\n\n print('Processing Test EEG parquets...'); print()\n for i,eeg_id in enumerate(EEG_IDS2):\n \n # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n data = eeg_from_parquet(f'{PATH2}/{eeg_id}.parquet')\n all_raw_eegs2[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:30:21.657770Z","iopub.execute_input":"2024-04-06T21:30:21.658421Z","iopub.status.idle":"2024-04-06T21:30:21.676064Z","shell.execute_reply.started":"2024-04-06T21:30:21.658393Z","shell.execute_reply":"2024-04-06T21:30:21.675041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission ON TEST\nif SUBMISSION:\n preds = []\n params = {'data':test,'mode':'test','specs':spectrograms2, 'eeg_specs':all_eegs2, 'raw_eegs':all_raw_eegs2} \n models = MODELS if ENSEMBLE else MODEL\n for i in range(5):\n print(f'Fold {i+1}')\n pred = predict(models,params,i)\n preds.append(pred)\n \n pred = np.mean(preds,axis=0)\n print('Test preds shape',pred.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-06T21:30:40.330282Z","iopub.execute_input":"2024-04-06T21:30:40.330650Z","iopub.status.idle":"2024-04-06T21:30:49.182672Z","shell.execute_reply.started":"2024-04-06T21:30:40.330613Z","shell.execute_reply":"2024-04-06T21:30:49.181234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMISSION:\n sub = pd.DataFrame({'eeg_id':test.eeg_id.values})\n sub[TARGETS] = pred\n sub.to_csv('submission.csv',index=False)\n print('Submissionn shape',sub.shape)\n print()\n print(sub.head().to_string())","metadata":{},"execution_count":null,"outputs":[]}]}