{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Few ideas inspired from https://www.kaggle.com/code/ttahara/hms-hbac-resnet34d-baseline-training","metadata":{}},{"cell_type":"markdown","source":"### Upvote if it helps","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport pandas as pd\nimport numpy as np\nimport random\n\nfrom sklearn import model_selection\n\nimport tensorflow\n\nimport torch\nfrom torch.utils.data import DataLoader,Dataset\nimport torch.nn as nn\nfrom torch import optim\nimport torch.functional as F\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\n\nfrom torchvision import transforms\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\n# from tqdm import tqdm\nfrom tqdm.auto import tqdm\nimport torchaudio\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-20T15:56:03.625062Z","iopub.execute_input":"2024-01-20T15:56:03.625464Z","iopub.status.idle":"2024-01-20T15:56:03.633159Z","shell.execute_reply.started":"2024-01-20T15:56:03.625432Z","shell.execute_reply":"2024-01-20T15:56:03.632117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nSRC = ROOT / \"src\"\n\nDATA = INPUT / \"hms-harmful-brain-activity-classification\"\nTRAIN_SPEC = DATA / \"train_spectrograms\"\nTEST_SPEC = DATA / \"test_spectrograms\"\n\nTMP = ROOT / \"tmp\"\nTRAIN_SPEC_SPLIT = TMP / \"train_spectrograms_split\"\nTEST_SPEC_SPLIT = TMP / \"test_spectrograms_split\"\n\nTMP.mkdir(exist_ok=True)\nTRAIN_SPEC_SPLIT.mkdir(exist_ok=True)\nTEST_SPEC_SPLIT.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:04.354818Z","iopub.execute_input":"2024-01-20T15:56:04.355691Z","iopub.status.idle":"2024-01-20T15:56:04.361967Z","shell.execute_reply.started":"2024-01-20T15:56:04.355660Z","shell.execute_reply":"2024-01-20T15:56:04.360999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    CLASSES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\n    N_CLASSES = len(CLASSES)\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    image_transform=transforms.Resize((256, 256))\n    t_transform = torchaudio.transforms.Spectrogram()\n    SEED=1086\n    N_FOLDS=5\n    NUM_EPOCHS=8    \n    BATCH_SIZE=32\n    NUM_WORKERS=4\n    PATIENCE=3\n    EPS=1e-5\ncfg = CFG()","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:04.963776Z","iopub.execute_input":"2024-01-20T15:56:04.964109Z","iopub.status.idle":"2024-01-20T15:56:04.971451Z","shell.execute_reply.started":"2024-01-20T15:56:04.964083Z","shell.execute_reply":"2024-01-20T15:56:04.970355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n    random.seed(seed)\nseed_everything(cfg.SEED)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:05.695761Z","iopub.execute_input":"2024-01-20T15:56:05.696103Z","iopub.status.idle":"2024-01-20T15:56:05.702277Z","shell.execute_reply.started":"2024-01-20T15:56:05.696079Z","shell.execute_reply":"2024-01-20T15:56:05.701425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(DATA / \"train.csv\")\n# convert vote to probability\ntrain[cfg.CLASSES] /= train[cfg.CLASSES].sum(axis=1).values[:, None]\ntrain = train.groupby(\"spectrogram_id\").head(1).reset_index(drop=True)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:10.361549Z","iopub.execute_input":"2024-01-20T15:56:10.362366Z","iopub.status.idle":"2024-01-20T15:56:10.571390Z","shell.execute_reply.started":"2024-01-20T15:56:10.362334Z","shell.execute_reply":"2024-01-20T15:56:10.570405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train.sample(100).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:11.857971Z","iopub.execute_input":"2024-01-20T15:56:11.858353Z","iopub.status.idle":"2024-01-20T15:56:11.865200Z","shell.execute_reply.started":"2024-01-20T15:56:11.858322Z","shell.execute_reply":"2024-01-20T15:56:11.864346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for spec_id, df in tqdm(train.groupby(\"spectrogram_id\")):\n    spec = pd.read_parquet(TRAIN_SPEC / f\"{spec_id}.parquet\")\n    spec_arr = spec.fillna(0).values[:, 1:].T.astype(\"float32\")\n    for spec_offset, label_id in df[[\"spectrogram_label_offset_seconds\", \"label_id\"]].astype(int).values:\n        spec_offset = spec_offset // 2\n        split_spec_arr = spec_arr[:, spec_offset: spec_offset + 300]\n        np.save(TRAIN_SPEC_SPLIT / f\"{label_id}.npy\" , split_spec_arr)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:14.988653Z","iopub.execute_input":"2024-01-20T15:56:14.989374Z","iopub.status.idle":"2024-01-20T15:56:18.193379Z","shell.execute_reply.started":"2024-01-20T15:56:14.989337Z","shell.execute_reply":"2024-01-20T15:56:18.192381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgkf = model_selection.StratifiedGroupKFold(n_splits=cfg.N_FOLDS, shuffle=True, random_state=cfg.SEED)\n\ntrain[\"kfold\"] = -1\n\nfor fold_id, (_, val_idx) in enumerate(\n    sgkf.split(train, y=train[\"expert_consensus\"], groups=train[\"patient_id\"])\n):\n    train.loc[val_idx, \"kfold\"] = fold_id","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:56:18.195230Z","iopub.execute_input":"2024-01-20T15:56:18.196009Z","iopub.status.idle":"2024-01-20T15:56:18.259673Z","shell.execute_reply.started":"2024-01-20T15:56:18.195972Z","shell.execute_reply":"2024-01-20T15:56:18.258766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HMSDataset(Dataset):\n    def __init__(self,df):\n        self.df = df\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx:int):\n        img_path = TRAIN_SPEC_SPLIT / f\"{self.df.loc[:,'label_id'][idx]}.npy\"\n        labels = self.df.loc[:,cfg.CLASSES].values[idx]\n        img = np.load(img_path)\n#         img = img - img.min()\n#         img = img / img.max()\n#         img= np.log(img)\n        data_mean=img.mean(axis=(0,1))\n        data_std=img.std(axis=(0,1))\n        img=(img-data_mean)/(data_std+cfg.EPS)\n        img = np.nan_to_num(img, nan=0.0)\n        data_tensor = torch.unsqueeze(torch.Tensor(img), dim=0)\n#         img = cfg.t_transform(img)\n        img = cfg.image_transform(data_tensor)\n        lab = [lab.astype(\"float32\") for lab in labels]\n        return torch.tensor(img),torch.tensor(lab)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T16:00:57.551643Z","iopub.execute_input":"2024-01-20T16:00:57.552000Z","iopub.status.idle":"2024-01-20T16:00:57.560864Z","shell.execute_reply.started":"2024-01-20T16:00:57.551972Z","shell.execute_reply":"2024-01-20T16:00:57.559864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HMSCnn(nn.Module):\n    def __init__(self):\n        super(HMSCnn, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)\n        self.relu3 = nn.ReLU()\n        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n        self.fc1 = nn.Linear(128 * 32 * 32, 256)\n        self.relu4 = nn.ReLU()\n        self.fc2 = nn.Linear(256, 128)\n        self.fc3 = nn.Linear(128, 6)\n\n    def forward(self, x):\n        x = self.pool1(self.relu1(self.conv1(x)))\n        x = self.pool2(self.relu2(self.conv2(x)))\n        x = self.pool3(self.relu3(self.conv3(x)))\n        x = x.view(-1, 128 * 32 * 32)\n        x = self.relu4(self.fc1(x))\n        x = self.fc2(x)\n        x = self.fc3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:00.337719Z","iopub.execute_input":"2024-01-20T15:57:00.338472Z","iopub.status.idle":"2024-01-20T15:57:00.350085Z","shell.execute_reply.started":"2024-01-20T15:57:00.338438Z","shell.execute_reply":"2024-01-20T15:57:00.349136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = HMSCnn()\nmodel = model.to(cfg.DEVICE)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:01.291765Z","iopub.execute_input":"2024-01-20T15:57:01.292669Z","iopub.status.idle":"2024-01-20T15:57:01.774545Z","shell.execute_reply.started":"2024-01-20T15:57:01.292632Z","shell.execute_reply":"2024-01-20T15:57:01.773540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class KLDivLossWithLogits(nn.KLDivLoss):\n\n    def __init__(self):\n        super().__init__(reduction=\"batchmean\")\n\n    def forward(self, y, t):\n        y = nn.functional.log_softmax(y,  dim=1)\n        loss = super().forward(y, t)\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:02.227433Z","iopub.execute_input":"2024-01-20T15:57:02.227811Z","iopub.status.idle":"2024-01-20T15:57:02.233680Z","shell.execute_reply.started":"2024-01-20T15:57:02.227783Z","shell.execute_reply":"2024-01-20T15:57:02.232571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# criterion = nn.MSELoss()\ncriterion = KLDivLossWithLogits()\noptimizer = optim.SGD(model.parameters(), lr=0.01)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:02.799489Z","iopub.execute_input":"2024-01-20T15:57:02.799915Z","iopub.status.idle":"2024-01-20T15:57:02.804682Z","shell.execute_reply.started":"2024-01-20T15:57:02.799877Z","shell.execute_reply":"2024-01-20T15:57:02.803809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n            path (str): Path for the checkpoint to be saved to.\n                            Default: 'checkpoint.pt'\n            trace_func (function): trace print function.\n                            Default: print            \n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n        self.path = path\n        self.trace_func = trace_func\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), self.path)\n        self.val_loss_min = val_loss","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:03.429256Z","iopub.execute_input":"2024-01-20T15:57:03.429653Z","iopub.status.idle":"2024-01-20T15:57:03.440391Z","shell.execute_reply.started":"2024-01-20T15:57:03.429625Z","shell.execute_reply":"2024-01-20T15:57:03.439492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fold(model,train_loader,valid_loader,patience,n_epochs,criterion,optimizer):\n    \n    train_losses = []\n    valid_losses = []\n    avg_train_losses = []\n    avg_valid_losses = []\n    \n    early_stopping = EarlyStopping(patience=patience, verbose=True)\n    \n    for epoch in tqdm(range(1,cfg.NUM_EPOCHS+1)):\n        \n        model.train()\n        for batch,(data,target) in enumerate(train_loader,1):\n            data,target = data.to(cfg.DEVICE),target.to(cfg.DEVICE)\n            optimizer.zero_grad()\n            \n            output = model(data)\n            loss = criterion(output,target)\n            \n            loss.backward()\n            optimizer.step()\n            \n            train_losses.append(loss.item())\n            \n        model.eval() \n        for data, target in valid_loader:\n            data,target = data.to(cfg.DEVICE),target.to(cfg.DEVICE)\n            output = model(data)\n            loss = criterion(output, target)\n            valid_losses.append(loss.item())\n            \n        train_loss = np.average(train_losses)\n        valid_loss = np.average(valid_losses)\n        avg_train_losses.append(train_loss)\n        avg_valid_losses.append(valid_loss)\n        \n        epoch_len = len(str(cfg.NUM_EPOCHS))\n        \n        print_msg = (f'[{epoch:>{epoch_len}}/{n_epochs:>{epoch_len}}] ' +\n                     f'train_loss: {train_loss:.5f} ' +\n                     f'valid_loss: {valid_loss:.5f}')\n        \n        print(print_msg)\n        \n        \n        train_losses = []\n        valid_losses = []\n        \n        early_stopping(valid_loss, model)\n        \n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break\n        \n    model.load_state_dict(torch.load('checkpoint.pt'))\n    \n    return  model, avg_train_losses, avg_valid_losses","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:03.981610Z","iopub.execute_input":"2024-01-20T15:57:03.981962Z","iopub.status.idle":"2024-01-20T15:57:03.993390Z","shell.execute_reply.started":"2024-01-20T15:57:03.981933Z","shell.execute_reply":"2024-01-20T15:57:03.992434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss(fold,train_loss,valid_loss):\n    fig = plt.figure(figsize=(5,5))\n    \n    plt.plot(range(1,len(train_loss)+1),train_loss, label='Training Loss')\n    plt.plot(range(1,len(valid_loss)+1),valid_loss,label='Validation Loss')\n    \n    minposs = valid_loss.index(min(valid_loss))+1 \n    plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')\n    \n    plt.title(f\"Plot for fold={fold}\")\n    plt.xlabel('epochs')\n    plt.ylabel('loss')\n    plt.ylim(0, 0.5) # consistent scale\n    plt.xlim(0, len(train_loss)+1) # consistent scale\n    plt.grid(True)\n    plt.legend()\n    plt.tight_layout()\n    plt.show()\n    fig.savefig('loss_plot.png', bbox_inches='tight')","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:04.474048Z","iopub.execute_input":"2024-01-20T15:57:04.474427Z","iopub.status.idle":"2024-01-20T15:57:04.482291Z","shell.execute_reply.started":"2024-01-20T15:57:04.474397Z","shell.execute_reply":"2024-01-20T15:57:04.481227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_on_folds(fold):\n    \n    train_df = train.query(\"kfold!=@fold\").reset_index(drop=True)\n    val_df = train.query(\"kfold==@fold\").reset_index(drop=True)\n    \n    trainset = HMSDataset(train_df)\n    train_loader = DataLoader(trainset,batch_size=cfg.BATCH_SIZE,num_workers=cfg.NUM_WORKERS,shuffle=True)\n\n    valset = HMSDataset(val_df)\n    val_loader = DataLoader(valset,batch_size=cfg.BATCH_SIZE,num_workers=cfg.NUM_WORKERS,shuffle=False)\n    \n    m,atl,avl = train_fold(model,\n                      train_loader,\n                      val_loader,\n                      cfg.PATIENCE,\n                      cfg.NUM_EPOCHS,\n                      criterion,\n                      optimizer)\n    \n    plot_loss(fold,atl,avl)\n    \n    torch.save(m.state_dict(),f\"hms_model_fold_{fold}.bin\")   ","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:04.951798Z","iopub.execute_input":"2024-01-20T15:57:04.952160Z","iopub.status.idle":"2024-01-20T15:57:04.959595Z","shell.execute_reply.started":"2024-01-20T15:57:04.952131Z","shell.execute_reply":"2024-01-20T15:57:04.958628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_on_folds(0)\ntrain_on_folds(1)\ntrain_on_folds(2)\ntrain_on_folds(3)\ntrain_on_folds(4)","metadata":{"execution":{"iopub.status.busy":"2024-01-20T15:57:05.518505Z","iopub.execute_input":"2024-01-20T15:57:05.518856Z","iopub.status.idle":"2024-01-20T15:57:06.626078Z","shell.execute_reply.started":"2024-01-20T15:57:05.518829Z","shell.execute_reply":"2024-01-20T15:57:06.624668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}