{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7402356,"sourceType":"datasetVersion","datasetId":4304475},{"sourceId":7403069,"sourceType":"datasetVersion","datasetId":4304949},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995},{"sourceId":7450712,"sourceType":"datasetVersion","datasetId":4336944},{"sourceId":7465251,"sourceType":"datasetVersion","datasetId":4317718},{"sourceId":7550085,"sourceType":"datasetVersion","datasetId":4397253},{"sourceId":7570342,"sourceType":"datasetVersion","datasetId":4407194},{"sourceId":7581697,"sourceType":"datasetVersion","datasetId":4413439},{"sourceId":7581715,"sourceType":"datasetVersion","datasetId":4413451},{"sourceId":7581720,"sourceType":"datasetVersion","datasetId":4413454},{"sourceId":7626715,"sourceType":"datasetVersion","datasetId":4382744},{"sourceId":7658551,"sourceType":"datasetVersion","datasetId":4417235},{"sourceId":158958765,"sourceType":"kernelVersion"},{"sourceId":159333316,"sourceType":"kernelVersion"},{"sourceId":159396114,"sourceType":"kernelVersion"},{"sourceId":162349255,"sourceType":"kernelVersion"}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The goal of this notebook is to combine some of the best public notebooks into one notebook, creating one ensembled submission file from a total of 4 notebooks.\n\nThe notebooks used are:\n\n- https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43\n\n- https://www.kaggle.com/code/yunsuxiaozi/hms-baseline-resnet34d-512-512-inference-6-models\n\n- https://www.kaggle.com/code/andreasbis/hms-inference-lb-0-41\n\n- https://www.kaggle.com/code/nartaa/features-head-starter-lb-0-36\n\nThis notebook is based on the work of JIYUANZHANG, found [here](https://www.kaggle.com/code/kitsuha/3-model-ensemble-lb-0-37)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport torch\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport timm\nfrom torch.utils.data import Dataset, DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.625808Z","iopub.execute_input":"2024-02-20T09:40:14.626277Z","iopub.status.idle":"2024-02-20T09:40:14.632195Z","shell.execute_reply.started":"2024-02-20T09:40:14.626241Z","shell.execute_reply":"2024-02-20T09:40:14.631235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n base_dir = pathlib.Path(\"/kaggle/input/hms-harmful-brain-activity-classification\")\n path_test = base_dir / \"test.csv\"\n path_submission = base_dir / \"sample_submission.csv\"\n spec_dir = base_dir / \"test_spectrograms\"\n model_name = \"tf_efficientnet_b0_ns\"\n model_weights = sorted(\n list(pathlib.Path(\"/kaggle/input/hms-pytorch-baseline-training-dataset\").glob(\"*.pt\"))\n + list(pathlib.Path(\"/kaggle/input/hms-pytorch-baseline-training\").glob(\"*.pt\"))\n )\n transform = transforms.Resize((512, 512), antialias=False)\n batch_size = 64\n label_columns = [\n \"seizure_vote\",\n \"lpd_vote\",\n \"gpd_vote\",\n \"lrda_vote\",\n \"grda_vote\",\n \"other_vote\",\n ]\n\n\nCFG.model_weights","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.643322Z","iopub.execute_input":"2024-02-20T09:40:14.643584Z","iopub.status.idle":"2024-02-20T09:40:14.656259Z","shell.execute_reply.started":"2024-02-20T09:40:14.643562Z","shell.execute_reply":"2024-02-20T09:40:14.655246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(CFG.path_test)\nsubmission = pd.read_csv(CFG.path_submission)\nsubmission = pd.merge(submission, test, how=\"inner\", on=\"eeg_id\")\nsubmission[\"path\"] = submission[\"spectrogram_id\"].map(lambda x: CFG.spec_dir / f\"{x}.parquet\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.858897Z","iopub.execute_input":"2024-02-20T09:40:14.859208Z","iopub.status.idle":"2024-02-20T09:40:14.883496Z","shell.execute_reply.started":"2024-02-20T09:40:14.859158Z","shell.execute_reply":"2024-02-20T09:40:14.882515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(x):\n x = np.clip(x, np.exp(-6), np.exp(10))\n x = np.log(x)\n m, s = x.mean(), x.std()\n x = (x - m) / (s + 1e-6)\n return x\n\n\nclass SpecDataset(Dataset):\n \n def __init__(self, df, transform=CFG.transform):\n self.df = df\n self.transform = transform\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n row = self.df.iloc[index]\n # input\n x = pd.read_parquet(row.path)\n x = x.fillna(-1).values[:, 1:].T\n x = preprocess(x)\n x = torch.Tensor(x[None, :])\n if self.transform:\n x = self.transform(x)\n # output\n y = np.array(row.loc[CFG.label_columns].values, 'float32')\n y = torch.Tensor(y)\n return x, y","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.88512Z","iopub.execute_input":"2024-02-20T09:40:14.885469Z","iopub.status.idle":"2024-02-20T09:40:14.895348Z","shell.execute_reply.started":"2024-02-20T09:40:14.885441Z","shell.execute_reply":"2024-02-20T09:40:14.894233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_ds = SpecDataset(df=submission)\ndata_loader = DataLoader(dataset=data_ds, num_workers=os.cpu_count())\ndata_loader","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.896704Z","iopub.execute_input":"2024-02-20T09:40:14.897077Z","iopub.status.idle":"2024-02-20T09:40:14.906275Z","shell.execute_reply.started":"2024-02-20T09:40:14.897042Z","shell.execute_reply":"2024-02-20T09:40:14.90522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = next(iter(data_loader))\nx.shape, x","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:14.90791Z","iopub.execute_input":"2024-02-20T09:40:14.908223Z","iopub.status.idle":"2024-02-20T09:40:15.270537Z","shell.execute_reply.started":"2024-02-20T09:40:14.908176Z","shell.execute_reply":"2024-02-20T09:40:15.269377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(x[0, 0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:15.272462Z","iopub.execute_input":"2024-02-20T09:40:15.272741Z","iopub.status.idle":"2024-02-20T09:40:15.628705Z","shell.execute_reply.started":"2024-02-20T09:40:15.272717Z","shell.execute_reply":"2024-02-20T09:40:15.627672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"DEVICE: {DEVICE}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:15.629996Z","iopub.execute_input":"2024-02-20T09:40:15.630305Z","iopub.status.idle":"2024-02-20T09:40:15.635525Z","shell.execute_reply.started":"2024-02-20T09:40:15.630278Z","shell.execute_reply":"2024-02-20T09:40:15.634503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = timm.create_model(model_name=CFG.model_name, pretrained=False, num_classes=6, in_chans=1)\nmodel.to(DEVICE)\nnum_parameter = sum(x.numel() for x in model.parameters())\nprint(f\"Model has {num_parameter} parameters.\")","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:15.637783Z","iopub.execute_input":"2024-02-20T09:40:15.63807Z","iopub.status.idle":"2024-02-20T09:40:15.758117Z","shell.execute_reply.started":"2024-02-20T09:40:15.638046Z","shell.execute_reply":"2024-02-20T09:40:15.757141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = pd.DataFrame(0.0, columns=CFG.label_columns, index=submission.index)\nfor i, path_weight in enumerate(CFG.model_weights):\n print(f\"Model {i}: {path_weight}\")\n model.load_state_dict(torch.load(path_weight))\n model.eval()\n with torch.no_grad():\n res = []\n for x, y in data_loader:\n x = x.to(DEVICE)\n pred = model(x)\n pred = F.softmax(pred, dim=1)\n pred = pred.detach().cpu().numpy()\n res.append(pred)\n res = np.concatenate(res)\n res = pd.DataFrame(res, columns=CFG.label_columns, index=submission.index)\n display(res)\n prediction = prediction + res\n print(\"\\n\")\nprediction = prediction / len(CFG.model_weights)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:15.759265Z","iopub.execute_input":"2024-02-20T09:40:15.759555Z","iopub.status.idle":"2024-02-20T09:40:20.838023Z","shell.execute_reply.started":"2024-02-20T09:40:15.75953Z","shell.execute_reply":"2024-02-20T09:40:20.836661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:20.840367Z","iopub.execute_input":"2024-02-20T09:40:20.840828Z","iopub.status.idle":"2024-02-20T09:40:20.855136Z","shell.execute_reply.started":"2024-02-20T09:40:20.840787Z","shell.execute_reply":"2024-02-20T09:40:20.854225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[CFG.label_columns] = prediction\nsubmission = submission[[\"eeg_id\"] + CFG.label_columns]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:20.856877Z","iopub.execute_input":"2024-02-20T09:40:20.857471Z","iopub.status.idle":"2024-02-20T09:40:20.873033Z","shell.execute_reply.started":"2024-02-20T09:40:20.857443Z","shell.execute_reply":"2024-02-20T09:40:20.87196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\ninput_size = 64\nhidden_size = 128 # Define el tamaño de la capa oculta\noutput_size = 10\nmodel = nn.Sequential(\n nn.Linear(input_size, hidden_size),\n nn.ReLU(),\n nn.Dropout(0.5), # Ejemplo de dropout\n nn.Linear(hidden_size, output_size)\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:20.874245Z","iopub.execute_input":"2024-02-20T09:40:20.874511Z","iopub.status.idle":"2024-02-20T09:40:20.884812Z","shell.execute_reply.started":"2024-02-20T09:40:20.874488Z","shell.execute_reply":"2024-02-20T09:40:20.884052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=None)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:20.886492Z","iopub.execute_input":"2024-02-20T09:40:20.88686Z","iopub.status.idle":"2024-02-20T09:40:20.893147Z","shell.execute_reply.started":"2024-02-20T09:40:20.886818Z","shell.execute_reply":"2024-02-20T09:40:20.892138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:20.898434Z","iopub.execute_input":"2024-02-20T09:40:20.89868Z","iopub.status.idle":"2024-02-20T09:40:21.936307Z","shell.execute_reply.started":"2024-02-20T09:40:20.89866Z","shell.execute_reply":"2024-02-20T09:40:21.935214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:21.937785Z","iopub.execute_input":"2024-02-20T09:40:21.938077Z","iopub.status.idle":"2024-02-20T09:40:21.944258Z","shell.execute_reply.started":"2024-02-20T09:40:21.938048Z","shell.execute_reply":"2024-02-20T09:40:21.943152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n seed=2024\n image_transform=transforms.Resize((512, 512))\n num_folds=5","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:21.94575Z","iopub.execute_input":"2024-02-20T09:40:21.946204Z","iopub.status.idle":"2024-02-20T09:40:21.953534Z","shell.execute_reply.started":"2024-02-20T09:40:21.946145Z","shell.execute_reply":"2024-02-20T09:40:21.95265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models=[]\nfor i in range(Config.num_folds):\n model = torch.load(f'/kaggle/input/hms-baseline-resnet34d-512-512-training-5-folds/HMS_resnet_fold{i}.pth')\n models.append(model)\nmodel = torch.load(\"/kaggle/input/hms-baseline-resnet34d-512-512-training/HMS_resnet.pth\")\nmodels.append(model)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:21.954673Z","iopub.execute_input":"2024-02-20T09:40:21.954931Z","iopub.status.idle":"2024-02-20T09:40:22.416147Z","shell.execute_reply.started":"2024-02-20T09:40:21.954908Z","shell.execute_reply":"2024-02-20T09:40:22.414902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n torch.backends.cudnn.deterministic = True#将cuda加速的随机数生成器设为确定性模式\n torch.backends.cudnn.benchmark = True#关闭CuDNN框架的自动寻找最优卷积算法的功能，以避免不同的算法对结果产生影响\n torch.manual_seed(seed)#pytorch的随机种子\n np.random.seed(seed)#numpy的随机种子\n random.seed(seed)#python内置的随机种子\nseed_everything(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:22.417533Z","iopub.execute_input":"2024-02-20T09:40:22.417823Z","iopub.status.idle":"2024-02-20T09:40:22.424401Z","shell.execute_reply.started":"2024-02-20T09:40:22.417799Z","shell.execute_reply":"2024-02-20T09:40:22.423157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\nsubmission=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\nsubmission=submission.merge(test_df,on='eeg_id',how='left')\nsubmission['path']=submission['spectrogram_id'].apply(lambda x: \"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/\"+str(x)+\".parquet\" )\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:22.425733Z","iopub.execute_input":"2024-02-20T09:40:22.42613Z","iopub.status.idle":"2024-02-20T09:40:22.452199Z","shell.execute_reply.started":"2024-02-20T09:40:22.426093Z","shell.execute_reply":"2024-02-20T09:40:22.451301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths=submission['path'].values\ntest_preds=[]\nfor path in paths:\n eps=1e-6\n data=pd.read_parquet(path)\n #这里最小值是0,故用-1填充.第一列是时间列,故去掉 ,行是不同列,列是时间\n data = data.fillna(-1).values[:,1:].T\n #选取一段时间的数据进行训练\n data=data[:,0:300]#(400,300)\n data=np.clip(data,np.exp(-6),np.exp(10))#最大值为89209464.0\n data= np.log(data)#对数变换\n #对数据进行归一化\n data_mean=data.mean(axis=(0,1))\n data_std=data.std(axis=(0,1))\n data=(data-data_mean)/(data_std+eps)\n data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n data=Config.image_transform(data_tensor)\n test_pred=[]\n for model in models:\n model.eval()\n with torch.no_grad():\n pred=F.softmax(model(data.unsqueeze(0)))[0]\n pred=pred.detach().cpu().numpy()\n test_pred.append(pred)\n test_pred=np.array(test_pred).mean(axis=0)\n test_preds.append(test_pred)\ntest_preds=np.array(test_preds)\ntest_preds","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:22.453304Z","iopub.execute_input":"2024-02-20T09:40:22.453575Z","iopub.status.idle":"2024-02-20T09:40:24.085105Z","shell.execute_reply.started":"2024-02-20T09:40:22.453551Z","shell.execute_reply":"2024-02-20T09:40:24.0841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub2=pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\nlabels=['seizure','lpd','gpd','lrda','grda','other']\nfor i in range(len(labels)):\n sub2[f'{labels[i]}_vote']=test_preds[:,i]\nsub2.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:24.08717Z","iopub.execute_input":"2024-02-20T09:40:24.087596Z","iopub.status.idle":"2024-02-20T09:40:24.105377Z","shell.execute_reply.started":"2024-02-20T09:40:24.087558Z","shell.execute_reply":"2024-02-20T09:40:24.104411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3","metadata":{}},{"cell_type":"code","source":"# Importing essential libraries\nimport gc\nimport os\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom IPython.display import display\n\n# PyTorch for deep learning\nimport timm\nimport torch\nimport torch.nn as nn \nimport torch.optim as optim\nimport torch.nn.functional as F\n\n# torchvision for image processing and augmentation\nimport torchvision.transforms as transforms\n\n# Suppressing minor warnings to keep the output clean\nwarnings.filterwarnings('ignore', category=Warning)\n\n# Reclaim memory no longer in use.\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:24.107278Z","iopub.execute_input":"2024-02-20T09:40:24.107685Z","iopub.status.idle":"2024-02-20T09:40:24.738584Z","shell.execute_reply.started":"2024-02-20T09:40:24.107648Z","shell.execute_reply":"2024-02-20T09:40:24.737602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n seed=42\n image_transform=transforms.Resize((512, 512))\n num_folds=5\n \n# Set the seed for reproducibility across multiple libraries\ndef set_seed(seed):\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n torch.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n \nset_seed(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:24.740141Z","iopub.execute_input":"2024-02-20T09:40:24.740483Z","iopub.status.idle":"2024-02-20T09:40:24.746402Z","shell.execute_reply.started":"2024-02-20T09:40:24.740457Z","shell.execute_reply":"2024-02-20T09:40:24.745227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load and store the trained models for each fold into a list\nmodels = []\n\n# Load ResNet34d\nfor i in range(Config.num_folds):\n # Create the same model architecture as during training\n model_resnet = timm.create_model('resnet34d', pretrained=False, num_classes=6, in_chans=1)\n \n # Load the trained weights from the corresponding file\n model_resnet.load_state_dict(torch.load(f'/kaggle/input/resnet34d/hms-train-resnet34d/resnet34d_fold{i}.pth', map_location=torch.device('cpu')))\n \n # Append the loaded model to the models list\n models.append(model_resnet)\n\n# Reclaim memory no longer in use.\ngc.collect()\n\n# Load EfficientNetB0\nfor j in range(Config.num_folds):\n # Create the same model architecture as during training\n model_effnet_b0 = timm.create_model('efficientnet_b0', pretrained=False, num_classes=6, in_chans=1)\n \n # Load the trained weights from the corresponding file\n model_effnet_b0.load_state_dict(torch.load(f'/kaggle/input/efficientnetb0/hms-train-efficientnetb0/efficientnet_b0_fold{j}.pth', map_location=torch.device('cpu')))\n \n # Append the loaded model to the models list\n models.append(model_effnet_b0)\n \n# Reclaim memory no longer in use.\ngc.collect()\n \n# Load EfficientNetB1\nfor k in range(Config.num_folds):\n # Create the same model architecture as during training\n model_effnet_b1 = timm.create_model('efficientnet_b1', pretrained=False, num_classes=6, in_chans=1)\n \n # Load the trained weights from the corresponding file\n model_effnet_b1.load_state_dict(torch.load(f'/kaggle/input/efficientnetb1/hms-train-efficientnetb1/efficientnet_b1_fold{k}.pth', map_location=torch.device('cpu')))\n \n # Append the loaded model to the models list\n models.append(model_effnet_b1)\n\n# Reclaim memory no longer in use.\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:24.747738Z","iopub.execute_input":"2024-02-20T09:40:24.74802Z","iopub.status.idle":"2024-02-20T09:40:29.519124Z","shell.execute_reply.started":"2024-02-20T09:40:24.747997Z","shell.execute_reply":"2024-02-20T09:40:29.51809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load test data and sample submission dataframe\ntest_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv\")\n\n# Merge the submission dataframe with the test data on EEG IDs\nsubmission = submission.merge(test_df, on='eeg_id', how='left')\n\n# Generate file paths for each spectrogram based on the EEG data in the submission dataframe\nsubmission['path'] = submission['spectrogram_id'].apply(lambda x: f\"/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/{x}.parquet\")\n\n# Display the first few rows of the submission dataframe\ndisplay(submission.head())\n\n# Reclaim memory no longer in use\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:29.520864Z","iopub.execute_input":"2024-02-20T09:40:29.521181Z","iopub.status.idle":"2024-02-20T09:40:29.987346Z","shell.execute_reply.started":"2024-02-20T09:40:29.521142Z","shell.execute_reply":"2024-02-20T09:40:29.986269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the weights for each model\nweight_resnet34d = 0.26\nweight_effnetb0 = 0.48\nweight_effnetb1 = 0.26\n\n# Get file paths for test spectrograms\npaths = submission['path'].values\ntest_predss = []\n\n# Generate predictions for each spectrogram using all models\nfor path in paths:\n eps = 1e-6\n # Read and preprocess spectrogram data\n data = pd.read_parquet(path)\n data = data.fillna(-1).values[:, 1:].T\n data = np.clip(data, np.exp(-6), np.exp(10))\n data = np.log(data)\n \n # Normalize the data\n data_mean = data.mean(axis=(0, 1))\n data_std = data.std(axis=(0, 1))\n data = (data - data_mean) / (data_std + eps)\n data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n data = Config.image_transform(data_tensor)\n\n test_pred = []\n \n # Generate predictions using all models\n for model in models:\n model.eval()\n with torch.no_grad():\n pred = F.softmax(model(data.unsqueeze(0)))[0]\n pred = pred.detach().cpu().numpy()\n test_pred.append(pred)\n \n # Combine predictions from all models using weighted voting\n weighted_pred = weight_resnet34d * np.mean(test_pred[:Config.num_folds], axis=0) + \\\n weight_effnetb0 * np.mean(test_pred[Config.num_folds:2*Config.num_folds], axis=0) + \\\n weight_effnetb1 * np.mean(test_pred[2*Config.num_folds:], axis=0)\n \n test_predss.append(weighted_pred)\n\n# Convert the list of predictions to a NumPy array for further processing\ntest_predss = np.array(test_predss)\n\n# Reclaim memory no longer in use\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:29.988851Z","iopub.execute_input":"2024-02-20T09:40:29.989273Z","iopub.status.idle":"2024-02-20T09:40:33.015501Z","shell.execute_reply.started":"2024-02-20T09:40:29.989235Z","shell.execute_reply":"2024-02-20T09:40:33.014577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predss","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.016958Z","iopub.execute_input":"2024-02-20T09:40:33.017391Z","iopub.status.idle":"2024-02-20T09:40:33.02493Z","shell.execute_reply.started":"2024-02-20T09:40:33.017353Z","shell.execute_reply":"2024-02-20T09:40:33.023853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 4","metadata":{}},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nimport tensorflow\nimport tensorflow.keras.backend as K\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.models import load_model\n\nLOAD_BACKBONE_FROM = '/kaggle/input/efficientnetb-tf-keras/EfficientNetB2.h5'\nLOAD_MODELS_FROM = '/kaggle/input/features-head-starter-models/'\nVER = 35\nDATA_TYPE = 'both' # both|eeg|kaggle|raw\nTEST_MODE = False\nsubmission = True\n\n# Setup for ensemble\nENSEMBLE = True\nLBs = [0.39,0.41,0.43,0.52] # for weighted ensemble we use LBs of each model\nVERK = 33.2 # Kaggle's spectrogram model version\nVERB = 35 # Kaggle's and EEG's spectrogram model version\nVERE = 34 # EEG's spectrogram model version\nVERR = 36 # EEG's raw wavenet model version\n\nnp.random.seed(42)\n\n# USE SINGLE GPU, MULTIPLE GPUS \ngpus = tf.config.list_physical_devices('GPU')\n# WE USE MIXED PRECISION\ntf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\nif len(gpus)>1:\n strategy = tf.distribute.MirroredStrategy()\n print(f'Using {len(gpus)} GPUs')\nelse:\n strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n print(f'Using {len(gpus)} GPU')","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.026436Z","iopub.execute_input":"2024-02-20T09:40:33.026809Z","iopub.status.idle":"2024-02-20T09:40:33.040895Z","shell.execute_reply.started":"2024-02-20T09:40:33.026773Z","shell.execute_reply":"2024-02-20T09:40:33.039942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\nFEATS2 = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\nFEAT2IDX = {x:y for x,y in zip(FEATS2,range(len(FEATS2)))}\n\ndef eeg_from_parquet(parquet_path):\n\n eeg = pd.read_parquet(parquet_path, columns=FEATS2)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n data = np.zeros((10_000,len(FEATS2)))\n for j,col in enumerate(FEATS2):\n \n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n \n data[:,j] = x\n\n return data\n\ndef add_kl(data):\n import torch\n labels = data[TARGETS].values + 1e-5\n\n # compute kl-loss with uniform distribution by pytorch\n data['kl'] = torch.nn.functional.kl_div(\n torch.log(torch.tensor(labels)),\n torch.tensor([1 / 6] * 6),\n reduction='none'\n ).sum(dim=1).numpy()\n return data\n\nif not submission:\n train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n TARGETS = ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']\n META = ['spectrogram_id','spectrogram_label_offset_seconds','patient_id','expert_consensus']\n train = train.groupby('eeg_id')[META+TARGETS\n ].agg({**{m:'first' for m in META},**{t:'sum' for t in TARGETS}}).reset_index() \n train[TARGETS] = train[TARGETS]/train[TARGETS].values.sum(axis=1,keepdims=True)\n train.columns = ['eeg_id','spec_id','offset','patient_id','target'] + TARGETS\n train = add_kl(train)\n print(train.head(1).to_string())","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.042514Z","iopub.execute_input":"2024-02-20T09:40:33.042927Z","iopub.status.idle":"2024-02-20T09:40:33.058041Z","shell.execute_reply.started":"2024-02-20T09:40:33.042895Z","shell.execute_reply":"2024-02-20T09:40:33.057025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif not submission:\n # FOR TESTING SET READ_FILES TO TRUE\n if TEST_MODE:\n train = train.sample(500,random_state=42).reset_index(drop=True)\n spectrograms = {}\n for i,e in enumerate(train.spec_id.values):\n if i%100==0: print(i,', ',end='')\n x = pd.read_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/{e}.parquet')\n spectrograms[e] = x.values\n all_eegs = {}\n for i,e in enumerate(train.eeg_id.values):\n if i%100==0: print(i,', ',end='')\n x = np.load(f'/kaggle/input/eeg-spectrograms/EEG_Spectrograms/{e}.npy')\n all_eegs[e] = x\n all_raw_eegs = {}\n for i,e in enumerate(train.eeg_id.values):\n if i%100==0: print(i,', ',end='')\n x = eeg_from_parquet(f'/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{e}.parquet') \n all_raw_eegs[e] = x\n else:\n spectrograms = None\n all_eegs = None\n all_raw_eegs = None\n if DATA_TYPE=='both' or DATA_TYPE=='kaggle':\n spectrograms = np.load('/kaggle/input/brain-spectrograms/specs.npy',allow_pickle=True).item()\n if DATA_TYPE=='both' or DATA_TYPE=='eeg':\n all_eegs = np.load('/kaggle/input/eeg-spectrograms/eeg_specs.npy',allow_pickle=True).item()\n if DATA_TYPE=='raw':\n all_raw_eegs = np.load('/kaggle/input/brain-eegs/eegs.npy',allow_pickle=True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.059551Z","iopub.execute_input":"2024-02-20T09:40:33.060482Z","iopub.status.idle":"2024-02-20T09:40:33.074391Z","shell.execute_reply.started":"2024-02-20T09:40:33.060446Z","shell.execute_reply":"2024-02-20T09:40:33.073459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as albu\nfrom scipy.signal import butter, lfilter\n\nclass DataGenerator():\n 'Generates data for Keras'\n def __init__(self, data, specs=None, eeg_specs=None, raw_eegs=None, augment=False, mode='train', data_type=DATA_TYPE): \n self.data = data\n self.augment = augment\n self.mode = mode\n self.data_type = data_type\n self.specs = specs\n self.eeg_specs = eeg_specs\n self.raw_eegs = raw_eegs\n self.on_epoch_end()\n \n def __len__(self):\n return self.data.shape[0]\n\n def __getitem__(self, index):\n X, y = self.data_generation(index)\n if self.augment: X = self.augmentation(X)\n return X, y\n \n def __call__(self):\n for i in range(self.__len__()):\n yield self.__getitem__(i)\n \n if i == self.__len__()-1:\n self.on_epoch_end()\n \n def on_epoch_end(self):\n if self.mode=='train': \n self.data = self.data.sample(frac=1).reset_index(drop=True)\n \n def data_generation(self, index):\n if self.data_type == 'both':\n X,y = self.generate_all_specs(index)\n elif self.data_type == 'eeg' or self.data_type == 'kaggle':\n X,y = self.generate_specs(index)\n elif self.data_type == 'raw':\n X,y = self.generate_raw(index)\n\n return X,y\n \n def generate_all_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n eeg = self.eeg_specs[row.eeg_id]\n spec = self.specs[row.spec_id]\n \n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in range(4)]\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n X[100+56:200+56,:256,0] = img[:,22:-22,2]\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n X[100+56:200+56,:256,1] = img[:,22:-22,3]\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n X[100+56:200+56,256:,0] = img[:,22:-22,1]\n X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n X[100+56:200+56,256:,1] = img[:,22:-22,3]\n \n # EEG\n img = eeg\n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n X[200+56:300+56,:256,0] = img[:,22:-22,0]\n X[300+56:400+56,:256,0] = img[:,22:-22,2]\n X[200+56:300+56,:256,1] = img[:,22:-22,1]\n X[300+56:400+56,:256,1] = img[:,22:-22,3]\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0]\n X[300+56:400+56,256:,0] = img[:,22:-22,1]\n X[200+56:300+56,256:,1] = img[:,22:-22,2]\n X[300+56:400+56,256:,1] = img[:,22:-22,3]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_specs(self, index):\n X = np.zeros((512,512,3),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n if self.mode=='test': \n offset = 0\n else:\n offset = int(row.offset/2)\n \n if self.data_type == 'eeg':\n img = self.eeg_specs[row.eeg_id]\n elif self.data_type == 'kaggle':\n spec = self.specs[row.spec_id]\n imgs = [spec[offset:offset+300,k*100:(k+1)*100].T for k in range(4)]\n img = np.stack(imgs,axis=-1)\n # LOG TRANSFORM SPECTROGRAM\n img = np.clip(img,np.exp(-4),np.exp(8))\n img = np.log(img)\n \n # STANDARDIZE PER IMAGE\n img = np.nan_to_num(img, nan=0.0) \n \n mn = img.flatten().min()\n mx = img.flatten().max()\n ep = 1e-5\n img = 255 * (img - mn) / (mx - mn + ep)\n \n X[0_0+56:100+56,:256,0] = img[:,22:-22,0]\n X[100+56:200+56,:256,0] = img[:,22:-22,2]\n X[0_0+56:100+56,:256,1] = img[:,22:-22,1]\n X[100+56:200+56,:256,1] = img[:,22:-22,3]\n \n X[0_0+56:100+56,256:,0] = img[:,22:-22,0]\n X[100+56:200+56,256:,0] = img[:,22:-22,1]\n X[0_0+56:100+56,256:,1] = img[:,22:-22,2]\n X[100+56:200+56,256:,1] = img[:,22:-22,3]\n \n X[200+56:300+56,:256,0] = img[:,22:-22,0]\n X[300+56:400+56,:256,0] = img[:,22:-22,2]\n X[200+56:300+56,:256,1] = img[:,22:-22,1]\n X[300+56:400+56,:256,1] = img[:,22:-22,3]\n \n X[200+56:300+56,256:,0] = img[:,22:-22,0]\n X[300+56:400+56,256:,0] = img[:,22:-22,1]\n X[200+56:300+56,256:,1] = img[:,22:-22,2]\n X[300+56:400+56,256:,1] = img[:,22:-22,3]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def generate_raw(self,index):\n X = np.zeros((10_000,8),dtype='float32')\n y = np.zeros((6,),dtype='float32')\n \n row = self.data.iloc[index]\n eeg = self.raw_eegs[row.eeg_id]\n \n # FEATURE ENGINEER\n X[:,0] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['T3']]\n X[:,1] = eeg[:,FEAT2IDX['T3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,2] = eeg[:,FEAT2IDX['Fp1']] - eeg[:,FEAT2IDX['C3']]\n X[:,3] = eeg[:,FEAT2IDX['C3']] - eeg[:,FEAT2IDX['O1']]\n \n X[:,4] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['C4']]\n X[:,5] = eeg[:,FEAT2IDX['C4']] - eeg[:,FEAT2IDX['O2']]\n \n X[:,6] = eeg[:,FEAT2IDX['Fp2']] - eeg[:,FEAT2IDX['T4']]\n X[:,7] = eeg[:,FEAT2IDX['T4']] - eeg[:,FEAT2IDX['O2']]\n \n # STANDARDIZE\n X = np.clip(X,-1024,1024)\n X = np.nan_to_num(X, nan=0) / 32.0\n \n # BUTTER LOW-PASS FILTER\n X = self.butter_lowpass_filter(X)\n # Downsample\n X = X[::5,:]\n \n if self.mode!='test':\n y[:] = row[TARGETS]\n \n return X,y\n \n def butter_lowpass_filter(self, data, cutoff_freq=20, sampling_rate=200, order=4):\n nyquist = 0.5 * sampling_rate\n normal_cutoff = cutoff_freq / nyquist\n b, a = butter(order, normal_cutoff, btype='low', analog=False)\n filtered_data = lfilter(b, a, data, axis=0)\n return filtered_data\n \n def resize(self, img,size):\n composition = albu.Compose([\n albu.Resize(size[0],size[1])\n ])\n return composition(image=img)['image']\n \n def augmentation(self, img):\n composition = albu.Compose([\n albu.HorizontalFlip(p=0.4)\n ])\n return composition(image=img)['image']","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.075865Z","iopub.execute_input":"2024-02-20T09:40:33.076215Z","iopub.status.idle":"2024-02-20T09:40:33.12728Z","shell.execute_reply.started":"2024-02-20T09:40:33.076164Z","shell.execute_reply":"2024-02-20T09:40:33.126345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submission and DATA_TYPE!='raw':\n gen = DataGenerator(train, augment=False, specs=spectrograms, eeg_specs=all_eegs, data_type=DATA_TYPE)\n for x,y in gen:\n break\n plt.imshow(x[:,:,0])\n plt.title(f'Target = {y.round(1)}',size=12)\n plt.yticks([])\n plt.ylabel('Frequencies (Hz)',size=12)\n plt.xlabel('Time (sec)',size=12)\n plt.show()\n \nif not submission and DATA_TYPE=='raw':\n gen = DataGenerator(train, raw_eegs=all_raw_eegs, data_type=DATA_TYPE)\n for x,y in gen:\n plt.figure(figsize=(20,4))\n offset = 0\n for j in range(x.shape[-1]):\n if j!=0: offset -= x[:,j].min()\n plt.plot(range(2_000),x[:,j]+offset,label=f'feature {j+1}')\n offset += x[:,j].max()\n plt.legend()\n plt.show()\n break","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.133903Z","iopub.execute_input":"2024-02-20T09:40:33.134201Z","iopub.status.idle":"2024-02-20T09:40:33.144421Z","shell.execute_reply.started":"2024-02-20T09:40:33.134149Z","shell.execute_reply":"2024-02-20T09:40:33.143271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submission:\n\n def lrfn(epoch):\n return [1e-3,1e-3,1e-4,1e-4][epoch]\n\n LR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n \n def lrfn2(epoch):\n return [1e-5,1e-5,1e-6][epoch]\n\n LR2 = tf.keras.callbacks.LearningRateScheduler(lrfn2, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.145647Z","iopub.execute_input":"2024-02-20T09:40:33.145934Z","iopub.status.idle":"2024-02-20T09:40:33.153208Z","shell.execute_reply.started":"2024-02-20T09:40:33.14591Z","shell.execute_reply":"2024-02-20T09:40:33.152281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate\n\ndef build_model(): \n inp = tf.keras.layers.Input((512,512,3))\n base_model = load_model(f'{LOAD_BACKBONE_FROM}') \n x = base_model(inp)\n x = tf.keras.layers.GlobalAveragePooling2D()(x)\n output = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(x)\n model = tf.keras.Model(inputs=inp, outputs=output)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer=opt) \n return model\n\ndef score(y_true, y_pred):\n kl = tf.keras.metrics.KLDivergence()\n return kl(y_true, y_pred)\n\ndef wave_block(x, filters, kernel_size, n):\n dilation_rates = [2**i for i in range(n)]\n x = Conv1D(filters = filters,\n kernel_size = 1,\n padding = 'same')(x)\n res_x = x\n for dilation_rate in dilation_rates:\n tanh_out = Conv1D(filters = filters,\n kernel_size = kernel_size,\n padding = 'same', \n activation = 'tanh', \n dilation_rate = dilation_rate)(x)\n sigm_out = Conv1D(filters = filters,\n kernel_size = kernel_size,\n padding = 'same',\n activation = 'sigmoid', \n dilation_rate = dilation_rate)(x)\n x = Multiply()([tanh_out, sigm_out])\n x = Conv1D(filters = filters,\n kernel_size = 1,\n padding = 'same')(x)\n res_x = Add()([res_x, x])\n return res_x\n\ndef build_wave_model():\n \n # INPUT \n inp = tf.keras.Input(shape=(2_000,8))\n \n ############\n # FEATURE EXTRACTION SUB MODEL\n inp2 = tf.keras.Input(shape=(2_000,1))\n x = wave_block(inp2, 8, 3, 12)\n x = wave_block(x, 16, 3, 8)\n x = wave_block(x, 32, 3, 4)\n x = wave_block(x, 64, 3, 1)\n model2 = tf.keras.Model(inputs=inp2, outputs=x)\n ###########\n \n # LEFT TEMPORAL CHAIN\n x1 = model2(inp[:,:,0:1])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,1:2])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z1 = tf.keras.layers.Average()([x1,x2])\n \n # LEFT PARASAGITTAL CHAIN\n x1 = model2(inp[:,:,2:3])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,3:4])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z2 = tf.keras.layers.Average()([x1,x2])\n \n # RIGHT PARASAGITTAL CHAIN\n x1 = model2(inp[:,:,4:5])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,5:6])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z3 = tf.keras.layers.Average()([x1,x2])\n \n # RIGHT TEMPORAL CHAIN\n x1 = model2(inp[:,:,6:7])\n x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n x2 = model2(inp[:,:,7:8])\n x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n z4 = tf.keras.layers.Average()([x1,x2])\n \n # COMBINE CHAINS\n y = tf.keras.layers.Concatenate()([z1,z2,z3,z4])\n y = tf.keras.layers.Dense(64, activation='relu')(y)\n y = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(y)\n \n # COMPILE MODEL\n model = tf.keras.Model(inputs=inp, outputs=y)\n opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n loss = tf.keras.losses.KLDivergence()\n model.compile(loss=loss, optimizer = opt)\n \n return model\n\ndef plot_hist(hist):\n metrics = ['loss']\n for i,metric in enumerate(metrics):\n plt.figure(figsize=(10,4))\n plt.subplot(1,2,i+1)\n plt.plot(hist[metric])\n plt.plot(hist[f'val_{metric}'])\n plt.title(f'{metric}',size=12)\n plt.ylabel(f'{metric}',size=12)\n plt.xlabel('epoch',size=12)\n plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.154449Z","iopub.execute_input":"2024-02-20T09:40:33.154772Z","iopub.status.idle":"2024-02-20T09:40:33.180537Z","shell.execute_reply.started":"2024-02-20T09:40:33.154742Z","shell.execute_reply":"2024-02-20T09:40:33.179622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nimport tensorflow.keras.backend as K, gc\n\nif not submission:\n all_oof = []\n all_true = []\n losses = []\n val_losses = []\n total_hist = {}\n\n gkf = GroupKFold(n_splits=5)\n for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)): \n \n print('#'*25)\n print(f'### Fold {i+1}')\n \n data, val = train.iloc[train_index],train.iloc[valid_index]\n train_gen = DataGenerator(data, augment=False, specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n valid_gen = DataGenerator(val, mode='valid', specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n data, val = data[data['kl']<5.5],val[val['kl']<5.5]\n train_gen2 = DataGenerator(data, augment=False, specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n valid_gen2 = DataGenerator(val, mode='valid', specs=spectrograms, eeg_specs=all_eegs, raw_eegs=all_raw_eegs)\n in_shape = (2000,8) if DATA_TYPE=='raw' else (512,512,3)\n EPOCHS = 4\n BATCH_SIZE_PER_REPLICA = 32\n BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n\n train_dataset = tf.data.Dataset.from_generator(generator=train_gen, \n output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n val_dataset = tf.data.Dataset.from_generator(generator=valid_gen, \n output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n train_dataset2 = tf.data.Dataset.from_generator(generator=train_gen2, \n output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n val_dataset2 = tf.data.Dataset.from_generator(generator=valid_gen2, \n output_signature=(tf.TensorSpec(shape=in_shape, dtype=tf.float32),\n tf.TensorSpec(shape=(6,), dtype=tf.float32))).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\n \n print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n print('#'*25)\n \n K.clear_session()\n with strategy.scope():\n if DATA_TYPE=='raw':\n model = build_wave_model()\n else:\n model = build_model()\n \n hist = model.fit(train_dataset, validation_data = val_dataset, \n epochs=EPOCHS, callbacks=[LR])\n print(f'### seconds stage train size {len(data)}, valid size {len(val)}')\n print('#'*25)\n hist2 = model.fit(train_dataset2, validation_data = val_dataset, \n epochs=3, callbacks=[LR2])\n losses.append(hist.history['loss']+hist2.history['loss'])\n val_losses.append(hist.history['val_loss']+hist2.history['val_loss'])\n K.clear_session()\n with strategy.scope():\n model.save_weights(f'model_{DATA_TYPE}_{VER}_{i}.weights.h5')\n oof = model.predict(val_dataset, verbose=1)\n all_oof.append(oof)\n all_true.append(train.iloc[valid_index][TARGETS].values) \n del model, oof\n gc.collect()\n \n total_hist['loss'] = np.mean(losses,axis=0)\n total_hist['val_loss'] = np.mean(val_losses,axis=0)\n all_oof = np.concatenate(all_oof)\n all_true = np.concatenate(all_true)\n plot_hist(total_hist)\n print('#'*25)\n print(f'CV KL SCORE: {score(all_true,all_oof)}')","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.181692Z","iopub.execute_input":"2024-02-20T09:40:33.182025Z","iopub.status.idle":"2024-02-20T09:40:33.204144Z","shell.execute_reply.started":"2024-02-20T09:40:33.181991Z","shell.execute_reply":"2024-02-20T09:40:33.203225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pywt, librosa\n\nUSE_WAVELET = None \n\nNAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n ['Fp1','F3','C3','P3','O1'],\n ['Fp2','F8','T4','T6','O2'],\n ['Fp2','F4','C4','P4','O2']]\n\n# DENOISE FUNCTION\ndef maddest(d, axis=None):\n return np.mean(np.absolute(d - np.mean(d, axis)), axis)\n\ndef denoise(x, wavelet='haar', level=1): \n coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n sigma = (1/0.6745) * maddest(coeff[-level])\n\n uthresh = sigma * np.sqrt(2*np.log(len(x)))\n coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n\n ret=pywt.waverec(coeff, wavelet, mode='per')\n \n return ret\n\nimport librosa\n\ndef spectrogram_from_eeg(parquet_path, display=False):\n \n # LOAD MIDDLE 50 SECONDS OF EEG SERIES\n eeg = pd.read_parquet(parquet_path)\n middle = (len(eeg)-10_000)//2\n eeg = eeg.iloc[middle:middle+10_000]\n \n # VARIABLE TO HOLD SPECTROGRAM\n img = np.zeros((100,300,4),dtype='float32')\n \n if display: plt.figure(figsize=(10,7))\n signals = []\n for k in range(4):\n COLS = FEATS[k]\n \n for kk in range(4):\n # FILL NANS\n x1 = eeg[COLS[kk]].values\n x2 = eeg[COLS[kk+1]].values\n m = np.nanmean(x1)\n if np.isnan(x1).mean()<1: x1 = np.nan_to_num(x1,nan=m)\n else: x1[:] = 0\n m = np.nanmean(x2)\n if np.isnan(x2).mean()<1: x2 = np.nan_to_num(x2,nan=m)\n else: x2[:] = 0\n \n # COMPUTE PAIR DIFFERENCES\n x = x1 - x2\n\n # DENOISE\n if USE_WAVELET:\n x = denoise(x, wavelet=USE_WAVELET)\n signals.append(x)\n\n # RAW SPECTROGRAM\n mel_spec = librosa.feature.melspectrogram(y=x, sr=200, hop_length=len(x)//300, \n n_fft=1024, n_mels=100, fmin=0, fmax=20, win_length=128)\n \n # LOG TRANSFORM\n width = (mel_spec.shape[1]//30)*30\n mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n img[:,:,k] += mel_spec_db\n \n # AVERAGE THE 4 MONTAGE DIFFERENCES\n img[:,:,k] /= 4.0\n \n if display:\n plt.subplot(2,2,k+1)\n plt.imshow(img[:,:,k],aspect='auto',origin='lower')\n \n if display: \n plt.show()\n plt.figure(figsize=(10,5))\n offset = 0\n for k in range(4):\n if k>0: offset -= signals[3-k].min()\n plt.plot(range(10_000),signals[k]+offset,label=NAMES[3-k])\n offset += signals[3-k].max()\n plt.legend()\n plt.show()\n \n return img","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.205421Z","iopub.execute_input":"2024-02-20T09:40:33.205742Z","iopub.status.idle":"2024-02-20T09:40:33.226665Z","shell.execute_reply.started":"2024-02-20T09:40:33.205713Z","shell.execute_reply":"2024-02-20T09:40:33.225584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if submission:\n test = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\n print('Test shape',test.shape)\n test.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.227756Z","iopub.execute_input":"2024-02-20T09:40:33.228081Z","iopub.status.idle":"2024-02-20T09:40:33.240878Z","shell.execute_reply.started":"2024-02-20T09:40:33.228057Z","shell.execute_reply":"2024-02-20T09:40:33.239971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL SPECTROGRAMS\nif submission:\n PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms/'\n files2 = os.listdir(PATH2)\n print(f'There are {len(files2)} test spectrogram parquets')\n \n spectrograms2 = {}\n for i,f in enumerate(files2):\n if i%100==0: print(i,', ',end='')\n tmp = pd.read_parquet(f'{PATH2}{f}')\n name = int(f.split('.')[0])\n spectrograms2[name] = tmp.iloc[:,1:].values\n \n # RENAME FOR DATA GENERATOR\n test = test.rename({'spectrogram_id':'spec_id'},axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T09:40:33.242354Z","iopub.execute_input":"2024-02-20T09:40:33.242707Z","iopub.status.idle":"2024-02-20T09:40:33.285971Z","shell.execute_reply.started":"2024-02-20T09:40:33.242674Z","shell.execute_reply":"2024-02-20T09:40:33.284972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ ALL EEG SPECTROGRAMS\nif submission:\n PATH2 = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\n DISPLAY = 0\n EEG_IDS2 = test.eeg_id.unique()\n all_eegs2 = {}\n\n print('Converting Test EEG to Spectrograms...'); print()\n for i,eeg_id in enumerate(EEG_IDS2):\n \n # CREATE SPECTROGRAM FROM EEG PARQUET\n img = spectrogram_from_eeg(f'{PATH2}{eeg_id}.parquet', i