{"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7465251,"sourceType":"datasetVersion","datasetId":4317718},{"sourceId":7517324,"sourceType":"datasetVersion","datasetId":4378712},{"sourceId":4533,"sourceType":"modelInstanceVersion","modelInstanceId":3325}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":5345.179582,"end_time":"2024-02-20T15:08:09.804733","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-20T13:39:04.625151","version":"2.4.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"088fc3f5e4264683bbeaa2d929fab599":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"danger","description":"","description_tooltip":null,"layout":"IPY_MODEL_2c3d333449fb412dbc86ca31835ef4b0","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3ba5491929cf4cb1bbb8e28514933aa6","value":1}},"11bfd3544cf8411e96cd7ede1451518c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5246ec5881c147f09c90846e9911dda6","placeholder":"​","style":"IPY_MODEL_db8fdfb810194e03b66c7c5e5077ccf5","value":" 1/? [00:00<00:00, 1.37it/s]"}},"2c3d333449fb412dbc86ca31835ef4b0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"3ba5491929cf4cb1bbb8e28514933aa6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5246ec5881c147f09c90846e9911dda6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58eb07f979fd49e8a9155ad60c77923c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"993085d825a542b1b5c44cd42f7f8c89":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c818523154ec45689615162b4c03bf96","IPY_MODEL_088fc3f5e4264683bbeaa2d929fab599","IPY_MODEL_11bfd3544cf8411e96cd7ede1451518c"],"layout":"IPY_MODEL_fa70a85db4944741a7d7b13cef19bb70"}},"c818523154ec45689615162b4c03bf96":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_58eb07f979fd49e8a9155ad60c77923c","placeholder":"​","style":"IPY_MODEL_ea4dcc5f4fcf4011a7ba7dc587109a9b","value":""}},"db8fdfb810194e03b66c7c5e5077ccf5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ea4dcc5f4fcf4011a7ba7dc587109a9b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"fa70a85db4944741a7d7b13cef19bb70":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport datetime as dt\nimport numpy as np\nimport pandas as pd\nimport wandb\n\nfrom glob import glob\nfrom pathlib import Path\nfrom typing import Dict, List\nfrom scipy.signal import butter, lfilter, freqz\nfrom matplotlib import pyplot as plt\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import (\n ReduceLROnPlateau,\n OneCycleLR,\n CosineAnnealingLR,\n CosineAnnealingWarmRestarts,\n)\nfrom torch.optim.optimizer import Optimizer\nfrom sklearn.model_selection import GroupKFold\n\nsys.path.append(\"/kaggle/input/kaggle-kl-div\")\nfrom kaggle_kl_div import score\n\n","metadata":{"papermill":{"duration":13.898864,"end_time":"2024-02-20T13:39:22.393405","exception":false,"start_time":"2024-02-20T13:39:08.494541","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:34.310294Z","iopub.execute_input":"2024-03-17T23:43:34.311282Z","iopub.status.idle":"2024-03-17T23:43:42.915554Z","shell.execute_reply.started":"2024-03-17T23:43:34.311243Z","shell.execute_reply":"2024-03-17T23:43:42.914518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Directory settings","metadata":{}},{"cell_type":"code","source":"class APP:\n jupyter = \"ipykernel\" in globals()\n if not jupyter:\n try:\n if \"IPython\" in globals().get(\"__doc__\", \"\"):\n jupyter = True\n except Exception as inst:\n print(inst)\n\n kaggle = os.environ.get(\"KAGGLE_KERNEL_RUN_TYPE\", \"\") != \"\"\n local = os.environ.get(\"DOCKER_USING\", \"\") == \"LOCAL\"\n date_time_start = dt.datetime.now()\n dt_start_ymd_hms = date_time_start.strftime(\"%Y.%m.%d_%H-%M-%S\")\n\n file_run_path = \"\"\n if jupyter:\n try:\n file_run_path = Path(globals().get(\"__vsc_ipynb_file__\", \"\"))\n except Exception as inst:\n print(inst)\n\n else:\n try:\n file_run_path = Path(__file__)\n except Exception as inst:\n print(inst)\n\n file_run_name = file_run_path.stem\n path_app = file_run_path.parent\n path_run = Path(os.getcwd())\n path_out = (\n Path(\"/kaggle/working\")\n if kaggle\n else file_run_path / f\"{file_run_name}_{dt_start_ymd_hms}\"\n )\n\nOUTPUT_DIR = \"./\"\nif not os.path.exists(OUTPUT_DIR):\n os.makedirs(OUTPUT_DIR)\n\nprint(f\"jupyter:{APP.jupyter}, kaggle:{APP.kaggle}, local:{APP.local}\")\nprint(APP.file_run_path)\nprint(APP.path_out)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:42.917323Z","iopub.execute_input":"2024-03-17T23:43:42.917774Z","iopub.status.idle":"2024-03-17T23:43:42.929144Z","shell.execute_reply.started":"2024-03-17T23:43:42.917748Z","shell.execute_reply":"2024-03-17T23:43:42.928224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n VERSION = 22\n\n wandb = False\n debug = False\n train = True\n create_eegs = False\n apex = True\n visualize = False\n save_all_models = True\n\n if debug:\n num_workers = 0\n parallel = False\n else:\n num_workers = os.cpu_count()\n parallel = True\n\n model_name = \"EEG\"\n # optimizer = \"Adan\"\n optimizer = \"AdamW\"\n\n factor = 0.9\n eps = 1e-6\n lr = 8e-3\n min_lr = 1e-6\n\n batch_size = 64\n weight_decay = 1e-2\n batch_scheduler = True\n gradient_accumulation_steps = 1\n max_grad_norm = 1e7\n\n fixed_kernel_size = 5\n # linear_layer_features = 424\n # kernels = [3, 5, 7, 9]\n linear_layer_features = 448\n kernels = [3, 5, 7, 9, 11]\n # kernels = [5, 7, 9, 11, 13]\n\n seq_length = 50 # Second's\n sampling_rate = 200 # Hz\n nsamples = seq_length * sampling_rate # Число семплов\n\n patience = 2\n n_fold = 3\n train_folds = [0, 1, 2]\n #train_folds = [0]\n seed = 2024\n\n total_evaluators = [(0, 6), (6, 1000)] # [(0, 10), (10, 1000)]\n epochs = [20, 20]\n\n filter_signal = True\n lowcut = 0.5 # 0.85 # Нижняя граница в Hz\n highcut = 20 # 25.0 # Верхняя граница в Hz\n filter_order = 2\n freq_channels = [] # [(8.0, 12.0)]; [(0.5, 4.5)]\n\n log_step = 100 # Шаг отображения тренировки\n log_show = False\n\n scheduler = \"CosineAnnealingWarmRestarts\" # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts','OneCycleLR']\n\n # CosineAnnealingLR params\n cosanneal_params = {\n \"T_max\": 6,\n \"eta_min\": 1e-5,\n \"last_epoch\": -1,\n }\n\n # ReduceLROnPlateau params\n reduce_params = {\n \"mode\": \"min\",\n \"factor\": 0.2,\n \"patience\": 4,\n \"eps\": 1e-6,\n \"verbose\": True,\n }\n\n # CosineAnnealingWarmRestarts params\n cosanneal_res_params = {\n \"T_0\": 20,\n \"eta_min\": 1e-6,\n \"T_mult\": 1,\n \"last_epoch\": -1,\n }\n\n target_cols = [\n \"seizure_vote\",\n \"lpd_vote\",\n \"gpd_vote\",\n \"lrda_vote\",\n \"grda_vote\",\n \"other_vote\",\n ]\n \n target_size = len(target_cols)\n\n pred_cols = [x + \"_pred\" for x in target_cols]\n\n map_features = [\n (\"Fp1\", \"T3\"),\n (\"T3\", \"O1\"),\n (\"Fp1\", \"C3\"),\n (\"C3\", \"O1\"),\n (\"Fp2\", \"C4\"),\n (\"C4\", \"O2\"),\n (\"Fp2\", \"T4\"),\n (\"T4\", \"O2\"),\n ]\n\n eeg_features = [\"Fp1\", \"T3\", \"C3\", \"O1\", \"Fp2\", \"C4\", \"T4\", \"O2\"]\n feature_to_index = {x: y for x, y in zip(eeg_features, range(len(eeg_features)))}\n\n # feature_to_index = {\n # row: i for i, row in enumerate({x for row in map_features for x in row})\n # }\n # eeg_features = [row for row in feature_to_index]\n\n # eeg_feat_size = len(eeg_features)\n n_map_features = len(map_features)\n in_channels = n_map_features + n_map_features * len(freq_channels)\n target_size = len(target_cols)\n\n path_inp = Path(\"/kaggle/input\")\n path_src = path_inp / \"hms-harmful-brain-activity-classification/\"\n file_train = path_src / \"train.csv\"\n path_train = path_src / \"train_eegs\"\n file_features_test = path_train / \"100261680.parquet\"\n file_eeg_specs = path_inp / \"eeg-spectrogram-by-lead-id-unique/eeg_specs.npy\"\n file_raw_eeg = path_inp / \"brain-eegs/eegs.npy\"\n\n if APP.kaggle:\n num_workers = 2\n parallel = True\n # GPU_DEVICES = \"auto\"\n\n\n# print(CFG.eeg_feat_size, CFG.in_channels)\nprint(CFG.feature_to_index)\nprint(CFG.eeg_features)","metadata":{"papermill":{"duration":0.029582,"end_time":"2024-02-20T13:39:22.435956","exception":false,"start_time":"2024-02-20T13:39:22.406374","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:42.930221Z","iopub.execute_input":"2024-03-17T23:43:42.930478Z","iopub.status.idle":"2024-03-17T23:43:42.947614Z","shell.execute_reply.started":"2024-03-17T23:43:42.930457Z","shell.execute_reply":"2024-03-17T23:43:42.946767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{"papermill":{"duration":0.012027,"end_time":"2024-02-20T13:39:22.461399","exception":false,"start_time":"2024-02-20T13:39:22.449372","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def init_logger(log_file=OUTPUT_DIR + \"train.log\"):\n from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n\n logger = getLogger(__name__)\n logger.setLevel(INFO)\n handler1 = StreamHandler()\n handler1.setFormatter(Formatter(\"%(message)s\"))\n handler2 = FileHandler(filename=log_file)\n handler2.setFormatter(Formatter(\"%(message)s\"))\n logger.addHandler(handler1)\n logger.addHandler(handler2)\n return logger\n\n\nLOGGER = init_logger()\n\ndef asMinutes(s):\n m = math.floor(s / 60)\n s -= m * 60\n return \"%dm %ds\" % (m, s)\n\n\ndef timeSince(since, percent):\n now = time.time()\n s = now - since\n es = s / (percent)\n rs = es - s\n return \"%s (remain %s)\" % (asMinutes(s), asMinutes(rs))\n\n\ndef quantize_data(data, classes):\n mu_x = mu_law_encoding(data, classes)\n return mu_x # quantized\n\n\ndef mu_law_encoding(data, mu):\n mu_x = np.sign(data) * np.log(1 + mu * np.abs(data)) / np.log(mu + 1)\n return mu_x\n\n\ndef mu_law_expansion(data, mu):\n s = np.sign(data) * (np.exp(np.abs(data) * np.log(mu + 1)) - 1) / mu\n return s\n\n\ndef butter_bandpass(lowcut, highcut, fs, order=5):\n return butter(order, [lowcut, highcut], fs=fs, btype=\"band\")\n\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n y = lfilter(b, a, data)\n return y\n\n\ndef butter_lowpass_filter(\n data, cutoff_freq=20, sampling_rate=CFG.sampling_rate, order=4\n):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n b, a = butter(order, normal_cutoff, btype=\"low\", analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n return filtered_data\n\n\ndef denoise_filter(x):\n # Частота дискретизации и желаемые частоты среза (в Гц).\n # Отфильтруйте шумный сигнал\n y = butter_bandpass_filter(x, CFG.lowcut, CFG.highcut, CFG.sampling_rate, order=6)\n y = (y + np.roll(y, -1) + np.roll(y, -2) + np.roll(y, -3)) / 4\n y = y[0:-1:4]\n return y","metadata":{"papermill":{"duration":0.045586,"end_time":"2024-02-20T13:39:22.519244","exception":false,"start_time":"2024-02-20T13:39:22.473658","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:42.949023Z","iopub.execute_input":"2024-03-17T23:43:42.949324Z","iopub.status.idle":"2024-03-17T23:43:42.965803Z","shell.execute_reply.started":"2024-03-17T23:43:42.949301Z","shell.execute_reply":"2024-03-17T23:43:42.964706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parquet to EEG Signals Numpy Processing","metadata":{"papermill":{"duration":0.011709,"end_time":"2024-02-20T13:40:46.512143","exception":false,"start_time":"2024-02-20T13:40:46.500434","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def eeg_from_parquet(\n parquet_path: str, display: bool = False, seq_length=CFG.seq_length\n) -> np.ndarray:\n \"\"\"\n Эта функция читает файл паркета и извлекает средние 50 секунд показаний. Затем он заполняет значения NaN\n со средним значением (игнорируя NaN).\n :param parquet_path: путь к файлу паркета.\n :param display: отображать графики ЭЭГ или нет.\n :return data: np.array формы (time_steps, eeg_features) -> (10_000, 8)\n \"\"\"\n\n # Вырезаем среднюю 50 секундную часть\n eeg = pd.read_parquet(parquet_path, columns=CFG.eeg_features)\n rows = len(eeg)\n\n # начало смещения данных, чтобы забрать середину\n offset = (rows - CFG.nsamples) // 2\n\n # средние 50 секунд, имеет одинаковое количество показаний слева и справа\n eeg = eeg.iloc[offset : offset + CFG.nsamples]\n\n if display:\n plt.figure(figsize=(10, 5))\n offset = 0\n\n # Конвертировать в numpy\n\n # создать заполнитель той же формы с нулями\n data = np.zeros((CFG.nsamples, len(CFG.eeg_features)))\n\n for index, feature in enumerate(CFG.eeg_features):\n x = eeg[feature].values.astype(\"float32\") # конвертировать в float32\n\n # Вычисляет среднее арифметическое вдоль указанной оси, игнорируя NaN.\n mean = np.nanmean(x)\n nan_percentage = np.isnan(x).mean() # percentage of NaN values in feature\n\n # Заполнение значения Nan\n # Поэлементная проверка на NaN и возврат результата в виде логического массива.\n if nan_percentage < 1: # если некоторые значения равны Nan, но не все\n x = np.nan_to_num(x, nan=mean)\n else: # если все значения — Nan\n x[:] = 0\n data[:, index] = x\n\n if display:\n if index != 0:\n offset += x.max()\n plt.plot(range(CFG.nsamples), x - offset, label=feature)\n offset -= x.min()\n\n if display:\n plt.legend()\n name = parquet_path.split(\"/\")[-1].split(\".\")[0]\n plt.yticks([])\n plt.title(f\"EEG {name}\", size=16)\n plt.show()\n\n return data","metadata":{"papermill":{"duration":0.026463,"end_time":"2024-02-20T13:40:46.550473","exception":false,"start_time":"2024-02-20T13:40:46.52401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:42.968974Z","iopub.execute_input":"2024-03-17T23:43:42.969305Z","iopub.status.idle":"2024-03-17T23:43:42.980907Z","shell.execute_reply.started":"2024-03-17T23:43:42.969283Z","shell.execute_reply":"2024-03-17T23:43:42.980023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.014802,"end_time":"2024-02-20T13:42:17.914507","exception":false,"start_time":"2024-02-20T13:42:17.899705","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class EEGDataset(Dataset):\n def __init__(\n self,\n df: pd.DataFrame,\n batch_size,\n eegs: Dict[int, np.ndarray],\n mode: str = \"train\",\n downsample: int = None,\n ):\n self.df = df\n self.batch_size = batch_size\n self.mode = mode\n self.eegs = eegs\n self.downsample = downsample\n\n def __len__(self):\n \"\"\"\n Length of dataset.\n \"\"\"\n # Обозначает количество пакетов за эпоху\n return len(self.df)\n\n def __getitem__(self, index):\n \"\"\"\n Get one item.\n \"\"\"\n # Сгенерировать один пакет данных\n X, y_prob = self.__data_generation(index)\n if self.downsample is not None:\n X = X[:: self.downsample, :]\n output = {\n \"eeg\": torch.tensor(X, dtype=torch.float32),\n \"labels\": torch.tensor(y_prob, dtype=torch.float32),\n }\n return output\n\n def __data_generation(self, index):\n # Генерирует данные, содержащие образцы размера партии\n row = self.df.iloc[index] # Строка Pandas\n X = np.zeros(\n (CFG.nsamples, CFG.in_channels), dtype=\"float32\"\n ) # Size=(10000, 14)\n y = np.zeros(CFG.target_size, dtype=\"float32\") # Size=(6,)\n data = self.eegs[row.eeg_id] # Size=(10000, 8)\n\n # === Feature engineering ===\n for i, (feat_a, feat_b) in enumerate(CFG.map_features):\n diff_feat = (\n data[:, CFG.feature_to_index[feat_a]]\n - data[:, CFG.feature_to_index[feat_b]]\n ) # Size=(10000,)\n\n if CFG.filter_signal:\n diff_feat = butter_bandpass_filter(\n diff_feat, CFG.lowcut, CFG.highcut, CFG.sampling_rate, order=CFG.filter_order # 6\n )\n\n if self.mode != \"test\" and random.uniform(0.0, 1.0) <= 0.1:\n lowcut = random.randint(10, 25) # random.uniform(1.0, 25.0)\n highcut = lowcut + 1.0 # random.uniform(0.0, 0.01)\n diff_feat = butter_bandpass_filter(\n diff_feat, lowcut, highcut, CFG.sampling_rate, order=CFG.filter_order # 6\n )\n\n X[:, i] = diff_feat\n\n if len(CFG.freq_channels) > 0:\n n = CFG.n_map_features\n for i in range(CFG.n_map_features):\n diff_feat = X[:, i]\n for j, (lowcut, highcut) in enumerate(CFG.freq_channels):\n band_feat = butter_bandpass_filter(\n diff_feat, lowcut, highcut, CFG.sampling_rate, order=CFG.filter_order, # 6\n )\n X[:, n] = band_feat\n n += 1\n\n # Обрезать края превышающие значения [-1024, 1024]\n X = np.clip(X, -1024, 1024)\n\n # Замените NaN нулем и разделить все на 32\n X = np.nan_to_num(X, nan=0) / 32.0\n\n # обрезать полосовым фильтром верхнюю границу в 20 HZ.\n X = butter_lowpass_filter(X, order=CFG.filter_order)\n \n y_prob = np.zeros(CFG.target_size, dtype=\"float32\")# 4\n\n if self.mode != \"test\":\n y_prob = row[CFG.target_cols].values.astype(np.float32)\n\n return X, y_prob","metadata":{"papermill":{"duration":0.0354,"end_time":"2024-02-20T13:42:17.964068","exception":false,"start_time":"2024-02-20T13:42:17.928668","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:42.982100Z","iopub.execute_input":"2024-03-17T23:43:42.982422Z","iopub.status.idle":"2024-03-17T23:43:42.999253Z","shell.execute_reply.started":"2024-03-17T23:43:42.982394Z","shell.execute_reply":"2024-03-17T23:43:42.998341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"class KLDivLossWithLogits(nn.KLDivLoss):\n def __init__(self):\n super().__init__(reduction=\"batchmean\")\n\n def forward(self, y, t):\n y = nn.functional.log_softmax(y, dim=1)\n loss = super().forward(y, t)\n return loss\n\nclass AverageMeter(object):\n \"\"\"Computes and stores the average and current value\"\"\"\n\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count\n\ndef seed_torch(seed):\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = True # Это опция требует много паямяти GPU\n # pl.seed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.000397Z","iopub.execute_input":"2024-03-17T23:43:43.000716Z","iopub.status.idle":"2024-03-17T23:43:43.012610Z","shell.execute_reply.started":"2024-03-17T23:43:43.000688Z","shell.execute_reply":"2024-03-17T23:43:43.011807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.031018,"end_time":"2024-02-20T13:42:28.75746","exception":false,"start_time":"2024-02-20T13:42:28.726442","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class ResNet_1D_Block(nn.Module):\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride,\n padding,\n downsampling,\n dilation=1,\n groups=1,\n dropout=0.0,\n ):\n super(ResNet_1D_Block, self).__init__()\n\n self.bn1 = nn.BatchNorm1d(num_features=in_channels)\n # self.relu = nn.ReLU(inplace=False)\n # self.relu_1 = nn.PReLU()\n # self.relu_2 = nn.PReLU()\n self.relu_1 = nn.Hardswish()\n self.relu_2 = nn.Hardswish()\n\n self.dropout = nn.Dropout(p=dropout, inplace=False)\n self.conv1 = nn.Conv1d(\n in_channels=in_channels,\n out_channels=out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n self.conv2 = nn.Conv1d(\n in_channels=out_channels,\n out_channels=out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.maxpool = nn.MaxPool1d(\n kernel_size=2,\n stride=2,\n padding=0,\n dilation=dilation,\n )\n self.downsampling = downsampling\n\n def forward(self, x):\n identity = x\n\n out = self.bn1(x)\n out = self.relu_1(out)\n out = self.dropout(out)\n out = self.conv1(out)\n out = self.bn2(out)\n out = self.relu_2(out)\n out = self.dropout(out)\n out = self.conv2(out)\n\n out = self.maxpool(out)\n identity = self.downsampling(x)\n\n out += identity\n return out\n\n\nclass EEGNet(nn.Module):\n def __init__(\n self,\n kernels,\n in_channels,\n fixed_kernel_size,\n num_classes,\n linear_layer_features,\n dilation=1,\n groups=1,\n ):\n super(EEGNet, self).__init__()\n self.kernels = kernels\n self.planes = 24\n self.parallel_conv = nn.ModuleList()\n self.in_channels = in_channels\n\n for i, kernel_size in enumerate(list(self.kernels)):\n sep_conv = nn.Conv1d(\n in_channels=in_channels,\n out_channels=self.planes,\n kernel_size=(kernel_size),\n stride=1,\n padding=0,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n self.parallel_conv.append(sep_conv)\n\n self.bn1 = nn.BatchNorm1d(num_features=self.planes)\n # self.relu = nn.ReLU(inplace=False)\n # self.relu_1 = nn.ReLU()\n # self.relu_2 = nn.ReLU()\n self.relu_1 = nn.SiLU()\n self.relu_2 = nn.SiLU()\n\n self.conv1 = nn.Conv1d(\n in_channels=self.planes,\n out_channels=self.planes,\n kernel_size=fixed_kernel_size,\n stride=2,\n padding=2,\n dilation=dilation,\n groups=groups,\n bias=False,\n )\n\n self.block = self._make_resnet_layer(\n kernel_size=fixed_kernel_size,\n stride=1,\n dilation=dilation,\n groups=groups,\n padding=fixed_kernel_size // 2,\n )\n self.bn2 = nn.BatchNorm1d(num_features=self.planes)\n self.avgpool = nn.AvgPool1d(kernel_size=6, stride=6, padding=2)\n\n self.rnn = nn.GRU(\n input_size=self.in_channels,\n hidden_size=128,\n num_layers=1,\n bidirectional=True,\n # dropout=0.2,\n )\n\n self.fc = nn.Linear(in_features=linear_layer_features, out_features=num_classes)\n\n def _make_resnet_layer(\n self,\n kernel_size,\n stride,\n dilation=1,\n groups=1,\n blocks=9,\n padding=0,\n dropout=0.0,\n ):\n layers = []\n downsample = None\n base_width = self.planes\n\n for i in range(blocks):\n downsampling = nn.Sequential(\n nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n )\n layers.append(\n ResNet_1D_Block(\n in_channels=self.planes,\n out_channels=self.planes,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n downsampling=downsampling,\n dilation=dilation,\n groups=groups,\n dropout=dropout,\n )\n )\n return nn.Sequential(*layers)\n\n def extract_features(self, x):\n x = x.permute(0, 2, 1)\n\n out_sep = []\n for i in range(len(self.kernels)):\n sep = self.parallel_conv[i](x)\n out_sep.append(sep)\n\n out = torch.cat(out_sep, dim=2)\n out = self.bn1(out)\n out = self.relu_1(out)\n out = self.conv1(out)\n\n out = self.block(out)\n out = self.bn2(out)\n out = self.relu_2(out)\n out = self.avgpool(out)\n\n out = out.reshape(out.shape[0], -1)\n rnn_out, _ = self.rnn(x.permute(0, 2, 1))\n new_rnn_h = rnn_out[:, -1, :]\n\n new_out = torch.cat([out, new_rnn_h], dim=1)\n return new_out\n\n def forward(self, x):\n new_out = self.extract_features(x)\n result = self.fc(new_out)\n return result","metadata":{"papermill":{"duration":0.05674,"end_time":"2024-02-20T13:42:28.844768","exception":false,"start_time":"2024-02-20T13:42:28.788028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:43.015233Z","iopub.execute_input":"2024-03-17T23:43:43.015603Z","iopub.status.idle":"2024-03-17T23:43:43.042128Z","shell.execute_reply.started":"2024-03-17T23:43:43.015579Z","shell.execute_reply":"2024-03-17T23:43:43.041421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adan Optimizer","metadata":{}},{"cell_type":"code","source":"class Adan(Optimizer):\n \"\"\"\n Implements a pytorch variant of Adan\n Adan was proposed in\n Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models[J]. arXiv preprint arXiv:2208.06677, 2022.\n https://arxiv.org/abs/2208.06677\n Arguments:\n params (iterable): iterable of parameters to optimize or dicts defining parameter groups.\n lr (float, optional): learning rate. (default: 1e-3)\n betas (Tuple[float, float, flot], optional): coefficients used for computing\n running averages of gradient and its norm. (default: (0.98, 0.92, 0.99))\n eps (float, optional): term added to the denominator to improve\n numerical stability. (default: 1e-8)\n weight_decay (float, optional): decoupled weight decay (L2 penalty) (default: 0)\n max_grad_norm (float, optional): value used to clip\n global grad norm (default: 0.0 no clip)\n no_prox (bool): how to perform the decoupled weight decay (default: False)\n \"\"\"\n\n def __init__(\n self,\n params,\n lr=1e-3,\n betas=(0.98, 0.92, 0.99),\n eps=1e-8,\n weight_decay=0.2,\n max_grad_norm=0.0,\n no_prox=False,\n ):\n if not 0.0 <= max_grad_norm:\n raise ValueError(\"Invalid Max grad norm: {}\".format(max_grad_norm))\n if not 0.0 <= lr:\n raise ValueError(\"Invalid learning rate: {}\".format(lr))\n if not 0.0 <= eps:\n raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n if not 0.0 <= betas[2] < 1.0:\n raise ValueError(\"Invalid beta parameter at index 2: {}\".format(betas[2]))\n defaults = dict(\n lr=lr,\n betas=betas,\n eps=eps,\n weight_decay=weight_decay,\n max_grad_norm=max_grad_norm,\n no_prox=no_prox,\n )\n super(Adan, self).__init__(params, defaults)\n\n def __setstate__(self, state):\n super(Adan, self).__setstate__(state)\n for group in self.param_groups:\n group.setdefault(\"no_prox\", False)\n\n @torch.no_grad()\n def restart_opt(self):\n for group in self.param_groups:\n group[\"step\"] = 0\n for p in group[\"params\"]:\n if p.requires_grad:\n state = self.state[p]\n # State initialization\n\n # Exponential moving average of gradient values\n state[\"exp_avg\"] = torch.zeros_like(p)\n # Exponential moving average of squared gradient values\n state[\"exp_avg_sq\"] = torch.zeros_like(p)\n # Exponential moving average of gradient difference\n state[\"exp_avg_diff\"] = torch.zeros_like(p)\n\n @torch.no_grad()\n def step(self):\n \"\"\"\n Performs a single optimization step.\n \"\"\"\n if self.defaults[\"max_grad_norm\"] > 0:\n device = self.param_groups[0][\"params\"][0].device\n global_grad_norm = torch.zeros(1, device=device)\n\n max_grad_norm = torch.tensor(self.defaults[\"max_grad_norm\"], device=device)\n for group in self.param_groups:\n\n for p in group[\"params\"]:\n if p.grad is not None:\n grad = p.grad\n global_grad_norm.add_(grad.pow(2).sum())\n\n global_grad_norm = torch.sqrt(global_grad_norm)\n\n clip_global_grad_norm = torch.clamp(\n max_grad_norm / (global_grad_norm + group[\"eps\"]), max=1.0\n )\n else:\n clip_global_grad_norm = 1.0\n\n for group in self.param_groups:\n beta1, beta2, beta3 = group[\"betas\"]\n # assume same step across group now to simplify things\n # per parameter step can be easily support by making it tensor, or pass list into kernel\n if \"step\" in group:\n group[\"step\"] += 1\n else:\n group[\"step\"] = 1\n\n bias_correction1 = 1.0 - beta1 ** group[\"step\"]\n bias_correction2 = 1.0 - beta2 ** group[\"step\"]\n bias_correction3 = 1.0 - beta3 ** group[\"step\"]\n\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n\n state = self.state[p]\n if len(state) == 0:\n state[\"exp_avg\"] = torch.zeros_like(p)\n state[\"exp_avg_sq\"] = torch.zeros_like(p)\n state[\"exp_avg_diff\"] = torch.zeros_like(p)\n\n grad = p.grad.mul_(clip_global_grad_norm)\n if \"pre_grad\" not in state or group[\"step\"] == 1:\n state[\"pre_grad\"] = grad\n\n copy_grad = grad.clone()\n\n exp_avg, exp_avg_sq, exp_avg_diff = (\n state[\"exp_avg\"],\n state[\"exp_avg_sq\"],\n state[\"exp_avg_diff\"],\n )\n diff = grad - state[\"pre_grad\"]\n\n update = grad + beta2 * diff\n exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) # m_t\n exp_avg_diff.mul_(beta2).add_(diff, alpha=1 - beta2) # diff_t\n exp_avg_sq.mul_(beta3).addcmul_(update, update, value=1 - beta3) # n_t\n\n denom = ((exp_avg_sq).sqrt() / math.sqrt(bias_correction3)).add_(\n group[\"eps\"]\n )\n update = (\n (\n exp_avg / bias_correction1\n + beta2 * exp_avg_diff / bias_correction2\n )\n ).div_(denom)\n\n if group[\"no_prox\"]:\n p.data.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update, alpha=-group[\"lr\"])\n else:\n p.add_(update, alpha=-group[\"lr\"])\n p.data.div_(1 + group[\"lr\"] * group[\"weight_decay\"])\n\n state[\"pre_grad\"] = copy_grad","metadata":{"papermill":{"duration":0.058157,"end_time":"2024-02-20T13:42:29.60282","exception":false,"start_time":"2024-02-20T13:42:29.544663","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:43:43.043404Z","iopub.execute_input":"2024-03-17T23:43:43.043714Z","iopub.status.idle":"2024-03-17T23:43:43.068374Z","shell.execute_reply.started":"2024-03-17T23:43:43.043686Z","shell.execute_reply":"2024-03-17T23:43:43.067505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train func","metadata":{}},{"cell_type":"code","source":"def train_fn(\n stage, fold, train_loader, model, criterion, optimizer, epoch, scheduler, device\n):\n model.train()\n\n scaler = torch.cuda.amp.GradScaler(enabled=CFG.apex)\n losses = AverageMeter()\n start = end = time.time()\n global_step = 0\n\n for step, batch in enumerate(train_loader):\n eegs = batch[\"eeg\"].to(device)\n labels = batch[\"labels\"].to(device)\n batch_size = labels.size(0)\n\n with torch.cuda.amp.autocast(enabled=CFG.apex):\n y_preds = model(eegs)\n loss = criterion(F.log_softmax(y_preds, dim=1), labels)\n\n if CFG.gradient_accumulation_steps > 1:\n loss = loss / CFG.gradient_accumulation_steps\n\n losses.update(loss.item(), batch_size)\n\n scaler.scale(loss).backward()\n\n grad_norm = torch.nn.utils.clip_grad_norm_(\n model.parameters(), CFG.max_grad_norm\n )\n\n if (step + 1) % CFG.gradient_accumulation_steps == 0:\n scaler.step(optimizer)\n scaler.update()\n global_step += 1\n if CFG.batch_scheduler:\n scheduler.step()\n end = time.time()\n\n if CFG.log_show and (step % CFG.log_step == 0 or step == (len(train_loader) - 1)):\n # remain=timeSince(start, float(step + 1) / len(train_loader))\n LOGGER.info(\n f\"Epoch {epoch+1} [{step}/{len(train_loader)}] Loss: {losses.val:.4f} Loss Avg:{losses.avg:.4f}\"\n )\n # \"Elapsed {remain:s} Grad: {grad_norm:.4f} LR: {cheduler.get_lr()[0]:.8f}\"\n\n if CFG.wandb:\n wandb.log(\n {\n f\"[fold{fold}] loss\": losses.val,\n f\"[fold{fold}] lr\": scheduler.get_lr()[0],\n }\n )\n return losses.avg","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.069453Z","iopub.execute_input":"2024-03-17T23:43:43.069704Z","iopub.status.idle":"2024-03-17T23:43:43.083711Z","shell.execute_reply.started":"2024-03-17T23:43:43.069683Z","shell.execute_reply":"2024-03-17T23:43:43.082821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Valid Func","metadata":{}},{"cell_type":"code","source":"def valid_fn(stage, epoch, valid_loader, model, criterion, device):\n losses = AverageMeter()\n model.eval()\n preds = []\n targets = []\n start = end = time.time()\n\n for step, batch in enumerate(valid_loader):\n eegs = batch[\"eeg\"].to(device)\n labels = batch[\"labels\"].to(device)\n batch_size = labels.size(0)\n\n with torch.no_grad():\n y_preds = model(eegs)\n loss = criterion(F.log_softmax(y_preds, dim=1), labels)\n\n if CFG.gradient_accumulation_steps > 1:\n loss = loss / CFG.gradient_accumulation_steps\n\n losses.update(loss.item(), batch_size)\n preds.append(nn.Softmax(dim=1)(y_preds).to(\"cpu\").numpy())\n targets.append(labels.to(\"cpu\").numpy())\n end = time.time()\n\n if CFG.log_show and (step % CFG.log_step == 0 or step == (len(valid_loader) - 1)):\n # remain=timeSince(start, float(step + 1) / len(valid_loader))\n LOGGER.info(\n f\"Epoch {epoch+1} VALIDATION: [{step}/{len(valid_loader)}] Val Loss: {losses.val:.4f} Val Loss Avg: {losses.avg:.4f}\"\n )\n # Elapsed {remain:s}\n\n predictions = np.concatenate(preds)\n targets = np.concatenate(targets)\n\n return losses.avg, predictions","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.084905Z","iopub.execute_input":"2024-03-17T23:43:43.085278Z","iopub.status.idle":"2024-03-17T23:43:43.097979Z","shell.execute_reply.started":"2024-03-17T23:43:43.085248Z","shell.execute_reply":"2024-03-17T23:43:43.097098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Optimizer","metadata":{"papermill":{"duration":0.031254,"end_time":"2024-02-20T13:42:29.808216","exception":false,"start_time":"2024-02-20T13:42:29.776962","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def build_optimizer(cfg, model, device, epochs, num_batches_per_epoch):\n lr = cfg.lr\n # lr = default_configs[\"lr\"]\n if cfg.optimizer == \"SAM\":\n base_optimizer = (\n torch.optim.SGD\n ) # define an optimizer for the \"sharpness-aware\" update\n optimizer_model = SAM(\n model.parameters(),\n base_optimizer,\n lr=lr,\n momentum=0.9,\n weight_decay=cfg.weight_decay,\n adaptive=True,\n )\n elif cfg.optimizer == \"Ranger21\":\n optimizer_model = Ranger21(\n model.parameters(),\n lr=lr,\n weight_decay=cfg.weight_decay,\n num_epochs=epochs,\n num_batches_per_epoch=num_batches_per_epoch\n )\n elif cfg.optimizer == \"SGD\":\n optimizer_model = torch.optim.SGD(\n model.parameters(), lr=lr, weight_decay=cfg.weight_decay, momentum=0.9\n )\n elif cfg.optimizer == \"Adam\":\n optimizer_model = Adam(model.parameters(), lr=lr, weight_decay=CFG.weight_decay)\n elif cfg.optimizer == \"AdamW\":\n optimizer_model = AdamW(\n model.parameters(), lr=lr, weight_decay=CFG.weight_decay\n )\n elif cfg.optimizer == \"Lion\":\n optimizer_model = Lion(model.parameters(), lr=lr, weight_decay=cfg.weight_decay)\n elif cfg.optimizer == \"Adan\":\n optimizer_model = Adan(model.parameters(), lr=lr, weight_decay=cfg.weight_decay)\n\n return optimizer_model","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.099310Z","iopub.execute_input":"2024-03-17T23:43:43.099920Z","iopub.status.idle":"2024-03-17T23:43:43.112020Z","shell.execute_reply.started":"2024-03-17T23:43:43.099889Z","shell.execute_reply":"2024-03-17T23:43:43.111196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scheduler","metadata":{}},{"cell_type":"code","source":"def get_scheduler(optimizer, epochs, steps_per_epoch):\n if CFG.scheduler == \"ReduceLROnPlateau\":\n scheduler = ReduceLROnPlateau(optimizer, **CFG.reduce_params)\n elif CFG.scheduler == \"CosineAnnealingLR\":\n scheduler = CosineAnnealingLR(optimizer, **CFG.cosanneal_params)\n elif CFG.scheduler == \"CosineAnnealingWarmRestarts\":\n scheduler = CosineAnnealingWarmRestarts(optimizer, **CFG.cosanneal_res_params)\n elif CFG.scheduler == \"OneCycleLR\":\n scheduler = OneCycleLR(\n optimizer=optimizer,\n epochs=epochs,\n pct_start=0.0,\n steps_per_epoch=steps_per_epoch,\n max_lr=CFG.lr,\n div_factor=25,\n final_div_factor=4.0e-01,\n )\n return scheduler","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.112917Z","iopub.execute_input":"2024-03-17T23:43:43.113182Z","iopub.status.idle":"2024-03-17T23:43:43.125527Z","shell.execute_reply.started":"2024-03-17T23:43:43.113160Z","shell.execute_reply":"2024-03-17T23:43:43.124607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hidden_size = 128# The dimension of the Transformer transformations: Key, Query, Scores and OUT\nnum_heads = 4# Number of Attention Heads\nnum_blocks = 1# The pooling applied before classification layer: 'avg','max' or 'cls'\ndropout_rate = .2# Drop rate\nchannels = 8\n\ndef positional_encoding(length, embed_dim):\n\n dim = embed_dim // 2\n\n position = np.arange(length)[:, np.newaxis]\n dim = np.arange(dim)[np.newaxis, :] / dim\n\n angle = 1 / (10000 ** dim)\n angle = position * angle\n\n pos_embed = np.concatenate([np.sin(angle), np.cos(angle)], axis=-1)\n pos_embed = torch.from_numpy(pos_embed).float()\n\n return pos_embed\n\n\nclass ConvBlock(nn.Module):\n\n def __init__(self, in_channels, out_channels):\n\n super(ConvBlock, self).__init__()\n\n self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=(8, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.bn1 = nn.BatchNorm2d(num_features=32)\n self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(8, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.bn2 = nn.BatchNorm2d(num_features=64)\n self.conv3 = nn.Conv2d(in_channels=64, out_channels=out_channels, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.activation = nn.LeakyReLU(inplace=False)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.activation(x)\n\n x = self.conv2(x)\n x = self.bn2(x)\n x = self.activation(x)\n\n x = self.conv3(x)\n\n return x\n\n\nclass MLP(nn.Module):\n\n def __init__(self, embed_dim, hidden_dim):\n\n super(MLP, self).__init__()\n\n self.mlp = nn.Sequential(\n nn.Linear(in_features=embed_dim, out_features=hidden_dim),\n nn.ReLU(inplace=True),\n nn.Linear(in_features=hidden_dim, out_features=embed_dim),\n )\n\n def forward(self, x):\n return self.mlp(x)\n\n\nclass TransformerBlock(nn.Module):\n\n def __init__(self, embed_dim, num_heads, out_dim):\n super(TransformerBlock, self).__init__()\n\n self.attention = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, batch_first=True)\n self.mlp = MLP(embed_dim=embed_dim, hidden_dim=out_dim)\n self.ln1 = nn.LayerNorm(embed_dim)\n self.ln2 = nn.LayerNorm(out_dim)\n\n def forward(self, x):\n\n x = self.ln1(x)\n x = self.attention(x, x, x)[0]\n x = self.ln2(x)\n x = self.mlp(x)\n\n return x\n\n\nclass HybridTransformer(nn.Module):\n\n def __init__(self, in_channels, hidden_size, num_heads, num_blocks, pooling_type, dropout_rate, *head_args):\n\n super(HybridTransformer, self).__init__()\n\n self.conv_block = ConvBlock(in_channels=1, out_channels=hidden_size)\n self.encoder = nn.ModuleList([\n TransformerBlock(\n embed_dim=hidden_size,\n num_heads=num_heads,\n out_dim=hidden_size,\n ) for _ in range(num_blocks)\n ])\n\n self.positional_embeddings = torch.nn.Parameter(positional_encoding(channels, hidden_size))\n self.cls_token = nn.Parameter(torch.zeros((1, hidden_size)))\n\n self.pooling_type = pooling_type\n self.dropout = nn.Dropout(dropout_rate) if dropout_rate > 0 else nn.Identity()\n#--------------------------------------------------------------------------------------------------\n self.head = nn.Linear(hidden_size,6)#ClassificationHead(input_dimensions=hidden_size, **head_args)\n#--------------------------------------------------------------------------------------------------\n def forward(self, x):\n\n # Add channel dimension and pass it to conv 2d block\n #[32, 1, 10000, 8]\n #[32, 1, 8, 10000]\n x = x.unsqueeze(dim=1).permute(0, 1, 2, 3) #changed permute 3, 2 to 2, 3 aka null\n x = self.conv_block(x)\n\n # Average features along time dimension\n x = torch.mean(x, dim=2).permute(0, 2, 1)\n\n # Add positional embeddings and concatenate cls token\n x += self.positional_embeddings\n x = torch.cat([\n self.cls_token.unsqueeze(0).repeat(x.size(0), 1, 1),\n x\n ], 1)\n\n # Pass it to transformer encoder\n for block in self.encoder:\n x = block(x)\n\n if self.pooling_type == 'avg':\n x = torch.mean(x, dim=1)\n elif self.pooling_type == 'max':\n x = torch.max(x, dim=1)[0]\n elif self.pooling_type == 'cls':\n x = x[:, 0, :]\n\n x = self.dropout(x)\n output = self.head(x)\n output = F.softmax(output, dim=-1)\n\n return output","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.130029Z","iopub.execute_input":"2024-03-17T23:43:43.130369Z","iopub.status.idle":"2024-03-17T23:43:43.157481Z","shell.execute_reply.started":"2024-03-17T23:43:43.130338Z","shell.execute_reply":"2024-03-17T23:43:43.156523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hidden_size = 128# The dimension of the Transformer transformations: Key, Query, Scores and OUT\nnum_heads = 4# Number of Attention Heads\nnum_blocks = 1# The pooling applied before classification layer: 'avg','max' or 'cls'\ndropout_rate = .2# Drop rate\nchannels = 8\n\n#img = torch.randn(32, 10000, 8)\n#m = HybridTransformer(channels, hidden_size, num_heads, num_blocks, \"cls\", dropout_rate)\n#print(m(img).shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.158642Z","iopub.execute_input":"2024-03-17T23:43:43.159031Z","iopub.status.idle":"2024-03-17T23:43:43.170561Z","shell.execute_reply.started":"2024-03-17T23:43:43.158997Z","shell.execute_reply":"2024-03-17T23:43:43.169673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoImageProcessor, AutoModel\nimport torch.nn as nn\n\nlearnable_modules = ['encoder.layer.0',\n 'encoder.layer.1',\n 'encoder.layer.2',\n 'encoder.layer.3']\nprocessor = AutoImageProcessor.from_pretrained('/kaggle/input/dinov2/pytorch/small/1')\ndinov2_vits14 = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/small/1')\n\nmodules = dict(dinov2_vits14.named_modules())\n#for m in learnable_modules:\n # modules[m].requires_grad_(False)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:43.171971Z","iopub.execute_input":"2024-03-17T23:43:43.172308Z","iopub.status.idle":"2024-03-17T23:43:59.888786Z","shell.execute_reply.started":"2024-03-17T23:43:43.172279Z","shell.execute_reply":"2024-03-17T23:43:59.887717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvBlock(nn.Module):\n\n def __init__(self, in_channels, out_channels):\n\n super(ConvBlock, self).__init__()\n\n self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.bn1 = nn.BatchNorm2d(num_features=32)\n self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.bn2 = nn.BatchNorm2d(num_features=64)\n self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(4, 1), stride=(2, 1), padding=(1, 0), padding_mode='zeros', bias=True)\n self.bn3 = nn.BatchNorm2d(num_features=128)\n self.out = nn.Conv2d(128, out_channels, 1, 1)\n self.activation = nn.LeakyReLU(inplace=False)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.activation(x)\n\n x = self.conv2(x)\n x = self.bn2(x)\n x = self.activation(x)\n\n x = self.conv3(x)\n x = self.bn3(x)\n x = self.activation(x)\n \n x = self.out(x) \n\n return x\n \ndef forward_hook(module, input, output):\n sequence_output = output[0]\n cls_token = sequence_output[:, 0]\n patch_tokens = sequence_output[:, 1:]\n global linear_input\n linear_input = cls_token\n return output\n \nclass DinoV2(nn.Module):\n def __init__(self):\n super().__init__()\n self.conv_encoder = ConvBlock(1, 3)\n self.encoder = dinov2_vits14\n self.classifier = nn.LazyLinear(6)\n def forward(self, x):\n x = x.unsqueeze(dim=1).permute(0, 1, 2, 3)\n B = x.size(0)\n x = self.conv_encoder(x).view(B, 3, 100, 100)\n hook = self.encoder.encoder.register_forward_hook(forward_hook)\n meta_logits = self.encoder(x)\n hook.remove()\n cls_logits = self.classifier(linear_input)\n return cls_logits","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:59.891358Z","iopub.execute_input":"2024-03-17T23:43:59.892591Z","iopub.status.idle":"2024-03-17T23:43:59.908303Z","shell.execute_reply.started":"2024-03-17T23:43:59.892544Z","shell.execute_reply":"2024-03-17T23:43:59.907480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def posemb_sincos_1d(patches, temperature = 10000, dtype = torch.float32):\n _, n, dim, device, dtype = *patches.shape, patches.device, patches.dtype\n\n n = torch.arange(n, device = device)\n assert (dim % 2) == 0, 'feature dimension must be multiple of 2 for sincos emb'\n omega = torch.arange(dim // 2, device = device) / (dim // 2 - 1)\n omega = 1. / (temperature ** omega)\n\n n = n.flatten()[:, None] * omega[None, :]\n pe = torch.cat((n.sin(), n.cos()), dim = 1)\n return pe.type(dtype)\n\nclass FeedForward(nn.Module):\n def __init__(self, dim, hidden_dim, dropout=0.2):\n super().__init__()\n self.net = nn.Sequential(\n nn.LayerNorm(dim),\n nn.Linear(dim, hidden_dim),\n nn.GELU(),\n nn.Linear(hidden_dim, dim),\n )\n def forward(self, x):\n return self.net(x)\n\nclass Attention(nn.Module):\n def __init__(self, dim, heads = 8, dim_head = 64, dropout=0.2):\n super().__init__()\n inner_dim = dim_head * heads\n self.dim_head = dim_head\n self.heads = heads\n self.scale = dim_head ** -0.5\n self.norm = nn.LayerNorm(dim)\n self.dropout = nn.Dropout(dropout)\n\n self.attend = nn.Softmax(dim = -1)\n\n self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n self.to_out = nn.Linear(inner_dim, dim, bias = False)\n\n def forward(self, x):\n B, S, _ = x.shape\n x = self.norm(x)\n\n qkv = self.to_qkv(x).chunk(3, dim = -1)\n q, k, v = map(lambda t: t.view(B, S, self.heads, self.dim_head).transpose(1, 2), qkv)\n\n dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n\n attn = self.attend(dots)\n attn = self.dropout(attn)\n\n out = torch.matmul(attn, v)\n out = out.view(B, S, -1)\n return self.to_out(out)\n\nclass Transformer(nn.Module):\n def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n super().__init__()\n self.layers = nn.ModuleList([])\n for _ in range(depth):\n self.layers.append(nn.ModuleList([\n Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout),\n FeedForward(dim, mlp_dim, dropout = dropout)\n ]))\n def forward(self, x):\n for attn, ff in self.layers:\n x = attn(x) + x\n x = ff(x) + x\n return x","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:59.909508Z","iopub.execute_input":"2024-03-17T23:43:59.909765Z","iopub.status.idle":"2024-03-17T23:43:59.928038Z","shell.execute_reply.started":"2024-03-17T23:43:59.909743Z","shell.execute_reply":"2024-03-17T23:43:59.927005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleViT(nn.Module):\n def __init__(self, *, seq_len, patch_size, num_classes, dim, depth, heads, mlp_dim, channels = 3, dim_head = 64):\n super().__init__()\n\n assert seq_len % patch_size == 0\n\n num_patches = seq_len // patch_size\n patch_dim = channels * patch_size\n\n\n self.to_patch_embedding = nn.Conv1d(channels, dim, patch_size, patch_size)\n self.cls_token = nn.Parameter(torch.randn(dim))\n self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim)\n\n self.to_latent = nn.Identity()\n self.linear_head = nn.Linear(dim, num_classes)\n\n def forward(self, series):\n series = series.permute(0, 2, 1)\n b, n, _ = series.shape\n\n x = self.to_patch_embedding(series).transpose(1, 2)\n x = torch.cat([torch.tile(self.cls_token, (b, 1, 1)),\n x], dim=1)\n \n x = x + torch.tile(self.pos_embedding, (b, 1, 1))\n x = self.transformer(x)\n cls_token = x[:, 0, :]\n\n x = self.to_latent(cls_token)\n return self.linear_head(x)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:59.929328Z","iopub.execute_input":"2024-03-17T23:43:59.929664Z","iopub.status.idle":"2024-03-17T23:43:59.942912Z","shell.execute_reply.started":"2024-03-17T23:43:59.929631Z","shell.execute_reply":"2024-03-17T23:43:59.942019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Loop","metadata":{}},{"cell_type":"code","source":"hidden_size = 128# The dimension of the Transformer transformations: Key, Query, Scores and OUT\nnum_heads = 4# Number of Attention Heads\nnum_blocks = 1# The pooling applied before classification layer: 'avg','max' or 'cls'\ndropout_rate = .2# Drop rate\nchannels = 8\n\nglobal chkps\nchkps = []\n\ndef train_loop(stage, epochs, folds, fold, directory, prev_dir, eggs):\n train_folds = folds[folds[\"fold\"] != fold].reset_index(drop=True)\n valid_folds = folds[folds[\"fold\"] == fold].reset_index(drop=True)\n valid_labels = valid_folds[CFG.target_cols].values\n\n train_dataset = EEGDataset(train_folds, batch_size=CFG.batch_size, mode=\"train\", eegs=eggs)\n valid_dataset = EEGDataset(valid_folds, batch_size=CFG.batch_size, mode=\"train\", eegs=eggs)\n\n train_loader = DataLoader(\n train_dataset,\n batch_size=CFG.batch_size,\n shuffle=True,\n num_workers=CFG.num_workers,\n pin_memory=True,\n drop_last=True,\n )\n\n valid_loader = DataLoader(\n valid_dataset,\n batch_size=CFG.batch_size * 2,\n shuffle=False,\n num_workers=CFG.num_workers,\n pin_memory=True,\n drop_last=False,\n )\n\n LOGGER.info(f\"========== stage: {stage} fold: {fold} training {len(train_loader)} / {len(valid_loader)} ==========\")\n\n #model = EEGNet(\n # kernels=CFG.kernels,\n # in_channels=CFG.in_channels,\n # fixed_kernel_size=CFG.fixed_kernel_size,\n #num_classes=CFG.target_size,\n #linear_layer_features=CFG.linear_layer_features,\n #)\n \n #model = HybridTransformer(channels, hidden_size, num_heads, num_blocks, \"cls\", dropout_rate)\n #model = SimpleViT(\n # seq_len = 10000,\n # patch_size = 625, #experiment ari\n # num_classes = 6,\n # dim = 1024,\n # depth = 1,\n # heads = 4,\n # mlp_dim = 2048,\n # channels = 8,\n # dim_head = 64\n # )\n model = nn.DataParallel(EEGNet(\n kernels=CFG.kernels,\n in_channels=CFG.in_channels,\n fixed_kernel_size=CFG.fixed_kernel_size,\n num_classes=CFG.target_size,\n linear_layer_features=CFG.linear_layer_features,\n ))\n \n if stage > 1:\n model_weight = (\n f\"{prev_dir}{CFG.model_name}_ver-{CFG.VERSION}_stage-{stage-1}_fold-{fold}_best.pth\"\n )\n checkpoint = torch.load(model_weight, map_location=device)\n model.load_state_dict(checkpoint[\"model\"])\n\n model.to(device)\n\n # CPMP: wrap the model to use all GPUs\n #if CFG.parallel:\n #model = nn.DataParallel(model)\n\n optimizer = build_optimizer(CFG, model, device, epochs=epochs, num_batches_per_epoch=len(train_loader))\n scheduler = get_scheduler(optimizer, epochs=epochs, steps_per_epoch=len(train_loader))\n criterion = nn.KLDivLoss(reduction=\"batchmean\")\n\n best_score = np.inf\n for epoch in range(epochs):\n start_time = time.time()\n\n # train\n avg_loss = train_fn(\n stage,\n fold,\n train_loader,\n model,\n criterion,\n optimizer,\n epoch,\n scheduler,\n device,\n )\n\n # eval\n avg_val_loss, predictions = valid_fn(\n stage,\n epoch,\n valid_loader,\n model,\n criterion,\n device,\n )\n\n elapsed = time.time() - start_time\n\n LOGGER.info(\n f\"Epoch {epoch+1} Avg Train Loss: {avg_loss:.4f} Avg Valid Loss: {avg_val_loss:.4f}\"\n )\n # time: {elapsed:.0f}s\n if CFG.wandb:\n wandb.log(\n {\n f\"[fold{fold}] stage\": stage,\n f\"[fold{fold}] epoch\": epoch + 1,\n f\"[fold{fold}] avg_train_loss\": avg_loss,\n f\"[fold{fold}] avg_val_loss\": avg_val_loss,\n f\"[fold{fold}] score\": score,\n }\n )\n\n if CFG.save_all_models:\n torch.save(\n {\"model\": model.state_dict(), \"predictions\": predictions}, #changed from model.module.state_dict() for nn.DataParallel\n f\"{directory}{CFG.model_name}_ver-{CFG.VERSION}_stage-{stage}_fold-{fold}_epoch-{epoch}_val-{avg_val_loss:.4f}_train-{avg_loss:.4f}.pth\",\n )\n\n if best_score > avg_val_loss:\n best_score = avg_val_loss\n LOGGER.info(\n f\"Epoch {epoch+1} Save Best Valid Loss: {avg_val_loss:.4f}\"\n )\n # CPMP: save the original model. It is stored as the module attribute of the DP model.\n torch.save(\n {\"model\": model.state_dict(), \"predictions\": predictions},\n f\"{directory}{CFG.model_name}_ver-{CFG.VERSION}_stage-{stage}_fold-{fold}_best.pth\",\n )\n chkps.append(model.state_dict())\n\n predictions = torch.load(\n f\"{directory}{CFG.model_name}_ver-{CFG.VERSION}_stage-{stage}_fold-{fold}_best.pth\",\n map_location=torch.device(\"cpu\"),\n )[\"predictions\"]\n\n # valid_folds[[f\"pred_{c}\" for c in CFG.target_cols]] = predictions\n valid_folds[CFG.pred_cols] = predictions\n valid_folds[CFG.target_cols] = valid_labels\n\n torch.cuda.empty_cache()\n gc.collect()\n\n return valid_folds, best_score","metadata":{"papermill":{"duration":0.05868,"end_time":"2024-02-20T13:42:29.897814","exception":false,"start_time":"2024-02-20T13:42:29.839134","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:50:33.141000Z","iopub.execute_input":"2024-03-17T23:50:33.141424Z","iopub.status.idle":"2024-03-17T23:50:33.164573Z","shell.execute_reply.started":"2024-03-17T23:50:33.141393Z","shell.execute_reply":"2024-03-17T23:50:33.163493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load train data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(CFG.file_train)\nTARGETS = train.columns[-6:]\nprint(\"Train shape:\", train.shape)\nprint(\"Targets\", list(TARGETS))\n\ntrain[\"total_evaluators\"] = train[CFG.target_cols].sum(axis=1)\n\ntrain_uniq = train.drop_duplicates(subset=[\"eeg_id\"] + list(TARGETS))\n\nprint(f\"There are {train.patient_id.nunique()} patients in the training data.\")\nprint(f\"There are {train.eeg_id.nunique()} EEG IDs in the training data.\")\nprint(f\"There are {train_uniq.shape[0]} unique eeg_id + votes in the training data.\")\n\nif CFG.visualize:\n train_uniq.eeg_id.value_counts().value_counts().plot(\n kind=\"bar\",\n title=f\"Distribution of Count of EEG w Unique Vote: \"\n f\"{train_uniq.shape[0]} examples\",\n )\n\ndel train_uniq\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:43:59.965014Z","iopub.execute_input":"2024-03-17T23:43:59.965354Z","iopub.status.idle":"2024-03-17T23:44:00.566503Z","shell.execute_reply.started":"2024-03-17T23:43:59.965324Z","shell.execute_reply":"2024-03-17T23:44:00.565463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.visualize:\n plt.figure(figsize=(10, 6))\n plt.hist(train[\"total_evaluators\"], bins=10, color=\"blue\", edgecolor=\"black\")\n plt.title(\"Histogram of Total Evaluators\")\n plt.xlabel(\"Total Evaluators\")\n plt.ylabel(\"Frequency\")\n plt.grid(True)\n plt.show()\n\ntst_eeg_df = pd.read_parquet(CFG.file_features_test)\ntst_eeg_features = tst_eeg_df.columns\nprint(f\"There are {len(tst_eeg_features)} raw eeg features\")\nprint(list(tst_eeg_features))\ndel tst_eeg_df\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:44:00.567703Z","iopub.execute_input":"2024-03-17T23:44:00.567998Z","iopub.status.idle":"2024-03-17T23:44:01.128638Z","shell.execute_reply.started":"2024-03-17T23:44:00.567972Z","shell.execute_reply":"2024-03-17T23:44:01.127601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Data / Deduplicate Train EEG Id","metadata":{}},{"cell_type":"code","source":"# %%time\nall_eeg_specs = np.load(CFG.file_eeg_specs, allow_pickle=True).item()\n\ntrain = train[train[\"label_id\"].isin(all_eeg_specs.keys())].copy()\nprint(train.shape[0])\n\ny_data = train[TARGETS].values + 0.166666667 # Regularization value\ny_data = y_data / y_data.sum(axis=1, keepdims=True)\ntrain[TARGETS] = y_data\n\ntrain[\"target\"] = train[\"expert_consensus\"]\n\ntrain_pops = []\nfor total_evaluator in CFG.total_evaluators:\n pop_idx = (train[\"total_evaluators\"] >= total_evaluator[0]) & (train[\"total_evaluators\"] < total_evaluator[1])\n train_pop = train[pop_idx].copy().reset_index()\n\n sgkf = GroupKFold(n_splits=CFG.n_fold)\n train_pop[\"fold\"] = -1\n for fold_id, (_, val_idx) in enumerate(\n sgkf.split(train_pop, y=train_pop[\"target\"], groups=train_pop[\"patient_id\"])\n ):\n train_pop.loc[val_idx, \"fold\"] = fold_id\n\n train_pops.append(train_pop)\n print(train_pop.shape[0])\n\nif CFG.visualize:\n print(\"Pop 1: train unique eeg_id + votes shape:\", train_pops[0].shape)\n plt.figure(figsize=(10, 6))\n plt.hist(train[\"total_evaluators\"], bins=10, color=\"blue\", edgecolor=\"black\")\n plt.title(\"Histogram of Total Evaluators\")\n plt.xlabel(\"Total Evaluators\")\n plt.ylabel(\"Frequency\")\n plt.grid(True)\n plt.show()\n\ndel all_eeg_specs\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:44:01.129803Z","iopub.execute_input":"2024-03-17T23:44:01.130085Z","iopub.status.idle":"2024-03-17T23:45:27.931247Z","shell.execute_reply.started":"2024-03-17T23:44:01.130049Z","shell.execute_reply":"2024-03-17T23:45:27.930275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\nif CFG.create_eegs:\n all_eegs = {}\n visualize = 1 if CFG.visualize else 0\n eeg_ids = train.eeg_id.unique()\n\n for i, eeg_id in tqdm(enumerate(eeg_ids)):\n\n # Сохранить ЭЭГ в словаре Python для массивов numpy\n eeg_path = CFG.path_train / f\"{eeg_id}.parquet\"\n\n # Вырезаем среднюю 50 секундную часть и заполняем по среднему Nan\n data = eeg_from_parquet(eeg_path, display=i < visualize)\n all_eegs[eeg_id] = data\n\n if i == visualize:\n if CFG.create_eegs:\n print(\n f\"Processing {train['eeg_id'].nunique()} eeg parquets... \", end=\"\"\n )\n else:\n print(f\"Reading {len(eeg_ids)} eeg NumPys from disk.\")\n break\n np.save(\"./eegs\", all_eegs)\n\nelse:\n all_eegs = np.load(CFG.file_raw_eeg, allow_pickle=True).item()\n\nif CFG.visualize:\n frequencies = [1, 2, 4, 8, 16][::-1] # frequencies in Hz\n x = [all_eegs[eeg_ids[0]][:, 0]] # select one EEG feature\n\n for frequency in frequencies:\n x.append(butter_lowpass_filter(x[0], cutoff_freq=frequency))\n\n plt.figure(figsize=(12, 8))\n plt.plot(range(CFG.nsamples), x[0], label=\"without filter\")\n for k in range(1, len(x)):\n plt.plot(\n range(CFG.nsamples),\n x[k] - k * (x[0].max() - x[0].min()),\n label=f\"with filter {frequencies[k-1]}Hz\",\n )\n\n plt.legend()\n plt.yticks([])\n plt.title(\"Butter Low-Pass Filter Examples\", size=18)\n plt.show() ","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:45:27.932497Z","iopub.execute_input":"2024-03-17T23:45:27.932849Z","iopub.status.idle":"2024-03-17T23:47:03.606764Z","shell.execute_reply.started":"2024-03-17T23:45:27.932816Z","shell.execute_reply":"2024-03-17T23:47:03.605889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = EEGDataset(train_pops[0], batch_size=CFG.batch_size, eegs=all_eegs, mode=\"train\")\ntrain_loader = DataLoader(\n train_dataset,\n batch_size=CFG.batch_size,\n shuffle=False,\n num_workers=CFG.num_workers,\n pin_memory=True,\n drop_last=True,\n)\noutput = train_dataset[0]\nX, y = output[\"eeg\"], output[\"labels\"]\nprint(f\"X shape: {X.shape}, y shape: {y.shape}\")\n\niot = torch.randn(32, CFG.nsamples, CFG.in_channels) # .cuda()\nmodel = EEGNet(\n kernels=CFG.kernels,\n in_channels=CFG.in_channels,\n fixed_kernel_size=CFG.fixed_kernel_size,\n num_classes=CFG.target_size,\n linear_layer_features=CFG.linear_layer_features,\n)\noutput = model(iot)\nprint(output.shape)\n\nif CFG.visualize:\n for batch in train_loader:\n X = batch.pop(\"eeg\")\n y = batch.pop(\"labels\")\n for item in range(4):\n plt.figure(figsize=(20, 4))\n offset = 0\n for col in range(X.shape[-1]):\n if col != 0:\n offset -= X[item, :, col].min()\n plt.plot(\n range(CFG.nsamples),\n X[item, :, col] + offset,\n label=f\"feature {col+1}\",\n )\n offset += X[item, :, col].max()\n tt = f\"{y[col][0]:0.1f}\"\n for t in y[col][1:]:\n tt += f\", {t:0.1f}\"\n plt.title(f\"EEG_Id = {eeg_ids[item]}\\nTarget = {tt}\", size=14)\n plt.legend()\n plt.show()\n break\n\ndel iot, model\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:47:03.607801Z","iopub.execute_input":"2024-03-17T23:47:03.608087Z","iopub.status.idle":"2024-03-17T23:47:08.215475Z","shell.execute_reply.started":"2024-03-17T23:47:03.608044Z","shell.execute_reply":"2024-03-17T23:47:08.214579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Stages","metadata":{}},{"cell_type":"code","source":"def train_process(stage, epochs, train_pop, eggs, pop_dir, prev_dir):\n oof_df = pd.DataFrame()\n \n if CFG.train:\n scores = []\n for fold in range(CFG.n_fold):\n if fold in CFG.train_folds:\n train_oof_df, score = train_loop(\n stage=stage,\n epochs=epochs,\n fold=fold,\n folds=train_pop,\n directory=pop_dir,\n prev_dir=prev_dir,\n eggs=eggs,\n )\n oof_df = pd.concat([oof_df, train_oof_df])\n scores.append(score)\n LOGGER.info(f\"========== stage: {stage} fold: {fold} result ==========\")\n LOGGER.info(f\"Score with best loss weights stage{stage}: {score:.4f}\")\n oof_df.reset_index(drop=True, inplace=True)\n LOGGER.info(f\"==================== CV ====================\")\n LOGGER.info(f\"Score with best loss weights: {np.mean(scores):.4f}\")\n oof_df.to_csv(\n f\"{pop_dir}{CFG.model_name}_oof_df_ver-{CFG.VERSION}_stage-{stage}.csv\",\n index=False,\n )\n\n if CFG.wandb:\n wandb.finish()\n \n return oof_df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:47:08.216899Z","iopub.execute_input":"2024-03-17T23:47:08.217199Z","iopub.status.idle":"2024-03-17T23:47:08.225673Z","shell.execute_reply.started":"2024-03-17T23:47:08.217175Z","shell.execute_reply":"2024-03-17T23:47:08.224689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#m = DinoV2().cuda()\nm = EEGNet(\n kernels=CFG.kernels,\n in_channels=CFG.in_channels,\n fixed_kernel_size=CFG.fixed_kernel_size,\n num_classes=CFG.target_size,\n linear_layer_features=CFG.linear_layer_features,\n )\nimg = torch.randn(64, 10000, 8)\nm(img).shape","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:47:08.226873Z","iopub.execute_input":"2024-03-17T23:47:08.227205Z","iopub.status.idle":"2024-03-17T23:47:18.049938Z","shell.execute_reply.started":"2024-03-17T23:47:08.227181Z","shell.execute_reply":"2024-03-17T23:47:18.049011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda:0\"\n\ndef get_score(preds, targets):\n oof = pd.DataFrame(preds.copy())\n oof[\"id\"] = np.arange(len(oof))\n true = pd.DataFrame(targets.copy())\n true[\"id\"] = np.arange(len(true))\n cv = score(solution=true, submission=oof, row_id_column_name=\"id\")\n return cv\n\n\ndef get_result(result_df):\n gt = result_df[[\"eeg_id\"] + CFG.target_cols]\n gt.sort_values(by=\"eeg_id\", inplace=True)\n gt.reset_index(inplace=True, drop=True)\n preds = result_df[[\"eeg_id\"] + CFG.pred_cols]\n preds.columns = [\"eeg_id\"] + CFG.target_cols\n preds.sort_values(by=\"eeg_id\", inplace=True)\n preds.reset_index(inplace=True, drop=True)\n score_loss = get_score(gt[CFG.target_cols], preds[CFG.target_cols])\n LOGGER.info(f\"Score with best loss weights: {score_loss}\")\n\n\nseed_torch(seed=CFG.seed)\n\nprev_dir = ''\nfor stage in range(len(CFG.total_evaluators)):\n pop_dir = f\"{OUTPUT_DIR}pop_{stage+1}_weight_oof/\"\n if not os.path.exists(pop_dir):\n os.makedirs(pop_dir)\n\n oof_df = train_process(\n stage=stage+1,\n epochs=10,\n train_pop=train_pops[stage],\n eggs=all_eegs,\n pop_dir=pop_dir,\n prev_dir=prev_dir,\n )\n #print(oof_df.head)\n prev_dir = pop_dir\n ","metadata":{"papermill":{"duration":2650.233098,"end_time":"2024-02-20T14:26:40.161837","exception":false,"start_time":"2024-02-20T13:42:29.928739","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:50:41.798701Z","iopub.execute_input":"2024-03-17T23:50:41.799079Z","iopub.status.idle":"2024-03-17T23:52:59.582098Z","shell.execute_reply.started":"2024-03-17T23:50:41.799038Z","shell.execute_reply":"2024-03-17T23:52:59.580279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"def inference_function(test_loader, model, device):\n model.eval() # set model in evaluation mode\n softmax = nn.Softmax(dim=1)\n prediction_dict = {}\n preds = []\n with tqdm(test_loader, unit=\"test_batch\", desc=\"Inference\") as tqdm_test_loader:\n for step, batch in enumerate(tqdm_test_loader):\n X = batch.pop(\"eeg\").to(device) # send inputs to `device`\n batch_size = X.size(0)\n with torch.no_grad():\n y_preds = model(X) # forward propagation pass\n y_preds = softmax(y_preds)\n preds.append(y_preds.to(\"cpu\").numpy()) # save predictions\n\n prediction_dict[\"predictions\"] = np.concatenate(\n preds\n ) # np.array() of shape (fold_size, target_cols)\n return prediction_dict","metadata":{"papermill":{"duration":0.167828,"end_time":"2024-02-20T15:08:05.860071","exception":false,"start_time":"2024-02-20T15:08:05.692243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-17T23:53:06.936098Z","iopub.execute_input":"2024-03-17T23:53:06.936467Z","iopub.status.idle":"2024-03-17T23:53:06.944123Z","shell.execute_reply.started":"2024-03-17T23:53:06.936441Z","shell.execute_reply":"2024-03-17T23:53:06.943141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\nprint(f\"Test dataframe shape is: {test_df.shape}\")\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:53:10.488814Z","iopub.execute_input":"2024-03-17T23:53:10.489216Z","iopub.status.idle":"2024-03-17T23:53:10.518825Z","shell.execute_reply.started":"2024-03-17T23:53:10.489179Z","shell.execute_reply":"2024-03-17T23:53:10.517757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_eeg_parquet_paths = glob(\"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\" + \"*.parquet\")\ntest_eeg_df = pd.read_parquet(test_eeg_parquet_paths[0])\ntest_eeg_features = test_eeg_df.columns\nprint(f\"There are {len(test_eeg_features)} raw eeg features\")\nprint(list(test_eeg_features))\ndel test_eeg_df\n_ = gc.collect()\n\n# %%time\nall_eegs = {}\neeg_ids = test_df.eeg_id.unique()\nfor i, eeg_id in tqdm(enumerate(eeg_ids)):\n # Save EEG to Python dictionary of numpy arrays\n eeg_path = \"/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/\" + str(eeg_id) + \".parquet\"\n data = eeg_from_parquet(eeg_path)\n all_eegs[eeg_id] = data","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:53:19.306935Z","iopub.execute_input":"2024-03-17T23:53:19.307705Z","iopub.status.idle":"2024-03-17T23:53:19.705005Z","shell.execute_reply.started":"2024-03-17T23:53:19.307672Z","shell.execute_reply":"2024-03-17T23:53:19.703881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"koef_sum = 0\nkoef_count = 0\npredictions = []\nfiles = []\n \n\ntest_dataset = EEGDataset(\n df=test_df,\n batch_size=32,\n mode=\"test\",\n eegs=all_eegs\n)\n\ntest_loader = DataLoader(\n test_dataset,\n batch_size=32,\n shuffle=False,\n num_workers=0,\n pin_memory=True,\n drop_last=False,\n)\nmodel = nn.DataParallel(EEGNet(\n kernels=CFG.kernels,\n in_channels=CFG.in_channels,\n fixed_kernel_size=CFG.fixed_kernel_size,\n num_classes=CFG.target_size,\n linear_layer_features=CFG.linear_layer_features,\n )).cuda()\nmodel.load_state_dict(chkps[-1])\n\npreds = []\n\nfor i in range(5):\n print('#'*25)\n print(f'### Testing Fold {i+1}')\n\n model.eval()\n fold_preds = []\n\n with torch.no_grad():\n for test_batch in test_loader:\n print(test_batch['eeg'].shape)\n test_batch = test_batch['eeg'].cuda()\n pred = model(test_batch).detach().cpu()\n pred = F.softmax(pred, dim=-1)\n print(pred)\n fold_preds.append(pred)\n fold_preds = np.concatenate(fold_preds)\n\n preds.append(fold_preds)\n\npred = np.mean(preds,axis=0)\nprint(\"pred\", pred)\nprint()\nprint('Test preds shape',pred.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:53:56.836871Z","iopub.execute_input":"2024-03-17T23:53:56.837269Z","iopub.status.idle":"2024-03-17T23:53:57.284223Z","shell.execute_reply.started":"2024-03-17T23:53:56.837241Z","shell.execute_reply":"2024-03-17T23:53:57.283044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({\"eeg_id\": test_df.eeg_id.values})\nsub[CFG.target_cols] = pred\n\nsub.to_csv(f\"submission.csv\", index=False)\nprint(f\"Submission shape: {sub.shape}\")\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T23:54:18.625782Z","iopub.execute_input":"2024-03-17T23:54:18.626177Z","iopub.status.idle":"2024-03-17T23:54:18.649009Z","shell.execute_reply.started":"2024-03-17T23:54:18.626147Z","shell.execute_reply":"2024-03-17T23:54:18.648004Z"},"trusted":true},"execution_count":null,"outputs":[]}]}