{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7392775,"sourceType":"datasetVersion","datasetId":4297782},{"sourceId":7447509,"sourceType":"datasetVersion","datasetId":4334995}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is written for educational purposes. I wanted to learn pytorch so I thought I'll implement this [notebook](https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43) by [Chris Deotte](https://www.kaggle.com/cdeotte). Thanks a lot Chris!","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport torchvision\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-07T17:44:59.310423Z","iopub.execute_input":"2024-02-07T17:44:59.310705Z","iopub.status.idle":"2024-02-07T17:45:05.670075Z","shell.execute_reply.started":"2024-02-07T17:44:59.310679Z","shell.execute_reply":"2024-02-07T17:45:05.669034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chris_spec_path = '/kaggle/input/brain-eeg-spectrograms/eeg_specs.npy'\nkaggle_spec_path = '/kaggle/input/brain-spectrograms/specs.npy'","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:45:05.671622Z","iopub.execute_input":"2024-02-07T17:45:05.672008Z","iopub.status.idle":"2024-02-07T17:45:05.676224Z","shell.execute_reply.started":"2024-02-07T17:45:05.671982Z","shell.execute_reply":"2024-02-07T17:45:05.675270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nchris_spec = np.load(chris_spec_path, allow_pickle = True).item()\nkaggle_spec = np.load(kaggle_spec_path, allow_pickle = True).item()","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:45:05.677228Z","iopub.execute_input":"2024-02-07T17:45:05.677453Z","iopub.status.idle":"2024-02-07T17:47:04.104116Z","shell.execute_reply.started":"2024-02-07T17:45:05.677433Z","shell.execute_reply":"2024-02-07T17:47:04.103173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ntargets = train_df.columns[-6:]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.106553Z","iopub.execute_input":"2024-02-07T17:47:04.106882Z","iopub.status.idle":"2024-02-07T17:47:04.420689Z","shell.execute_reply.started":"2024-02-07T17:47:04.106848Z","shell.execute_reply":"2024-02-07T17:47:04.419796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx2targets = {idx : target for idx, target in enumerate(targets)}\ntargets2idx = {target : idx for idx, target in enumerate(targets)}\ntargets2idx","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.421684Z","iopub.execute_input":"2024-02-07T17:47:04.422010Z","iopub.status.idle":"2024-02-07T17:47:04.428697Z","shell.execute_reply.started":"2024-02-07T17:47:04.421984Z","shell.execute_reply":"2024-02-07T17:47:04.427688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = train_df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = train_df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = train_df.groupby('eeg_id')[targets].agg('sum')\nfor t in targets:\n    train[t] = tmp[t].values\n    \ny_data = train[targets].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[targets] = y_data\n\ntmp = train_df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\nprint('Train non-overlapp eeg_id shape:', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.429719Z","iopub.execute_input":"2024-02-07T17:47:04.430025Z","iopub.status.idle":"2024-02-07T17:47:04.545745Z","shell.execute_reply.started":"2024-02-07T17:47:04.430001Z","shell.execute_reply":"2024-02-07T17:47:04.544788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HMSData(torch.utils.data.Dataset):\n    def __init__(self, kaggle_spec, chris_spec, df, mode):\n        super().__init__()\n        self.df = df\n        self.kaggle_spec = kaggle_spec\n        self.chris_spec = chris_spec\n        self.mode = mode\n        \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        \n        X = np.zeros((128, 256, 8), dtype = 'float32')\n        y = np.zeros(6, dtype = 'float32')\n        img = np.zeros((128, 265), dtype = 'float32')\n        \n        if self.mode == 'test':\n            r = 0\n        else:\n            r = int((row['max'] + row['min']) // 4) # each row in spectogram has a time difference of 2secs. Hence division by 4.\n        \n        for k in range(4): # that is, for LL, LR, RL, RR\n            img = self.kaggle_spec[row.spec_id][r : r + 300, k * 100 : (k + 1) * 100].T\n            \n            img = np.clip(img, np.exp(-4), np.exp(8))\n            img = np.log(img)\n            \n            ep = 1e-6\n            m = np.nanmean(img.flatten())\n            s = np.nanstd(img.flatten())\n            img = (img-m)/(s+ep)\n            img = np.nan_to_num(img, nan=0.0)\n            \n            X[14:-14, :, k] = img[:, 22:-22] / 2.0\n        \n        X[:, :, 4:] = self.chris_spec[row.eeg_id]\n        \n        if self.mode != 'test':\n            y = row[targets].values\n            y = y.astype(np.float32)\n            \n        X = torch.from_numpy(X)\n        y = torch.from_numpy(y)\n        \n        return X, y\n        \n    def __len__(self):\n        return self.df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.547117Z","iopub.execute_input":"2024-02-07T17:47:04.547789Z","iopub.status.idle":"2024-02-07T17:47:04.559971Z","shell.execute_reply.started":"2024-02-07T17:47:04.547735Z","shell.execute_reply":"2024-02-07T17:47:04.559080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        effnet = torchvision.models.efficientnet_b0(weights = 'DEFAULT')\n        modules = list(effnet.children())[:-1]\n        self.backbone = torch.nn.Sequential(*modules)\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n        \n        self.dropout = torch.nn.Dropout1d(0.2)\n        self.linear = torch.nn.Linear(effnet.classifier[1].in_features, 6)\n        self.softmax = torch.nn.Softmax(dim = 1)\n        \n    \n    def forward(self, inputs): # shape : (batch_size, 128, 256, 8)\n        x1 = [inputs[:, :, :, i : i + 1] for i in range(4)]\n        x1 = torch.cat(x1, dim = 1)\n        x2 = [inputs[:, :, :, i + 4 : i + 5] for i in range(4)]\n        x2 = torch.cat(x2, dim = 1)\n        x = torch.cat((x1, x2), dim = 2)\n        x = torch.cat((x, x, x), dim = 3) #shape : (batch_size, 512, 512, 3)\n        x = torch.permute(x, (0, 3, 1, 2)) #shape : (batch_size, 3, 512, 512)\n#         print(x.shape)\n        x = self.backbone(x)\n        batch_size = x.shape[0]\n        x = x.reshape(batch_size, -1)\n        x = self.dropout(x)\n        outputs = self.linear(x)\n#         outputs = self.softmax(x)\n#         print(outputs.shape)\n        return outputs        ","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.561411Z","iopub.execute_input":"2024-02-07T17:47:04.561690Z","iopub.status.idle":"2024-02-07T17:47:04.574384Z","shell.execute_reply.started":"2024-02-07T17:47:04.561657Z","shell.execute_reply":"2024-02-07T17:47:04.573517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\nk = 5\n\ndevice = ('cuda' if torch.cuda.is_available() else 'cpu')\nuse_amp = True\ncriterion = torch.nn.KLDivLoss(reduction = 'batchmean').to(device)\nsplits = GroupKFold(n_splits = k)\n\nLR_START = 1e-4\nLR_MAX = 1e-3\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 1\nLR_STEP_DECAY = 0.1\nEVERY = 1\nEPOCHS = 5\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = LR_MAX * LR_STEP_DECAY**((epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//EVERY)\n    return lr","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:47:04.575491Z","iopub.execute_input":"2024-02-07T17:47:04.575817Z","iopub.status.idle":"2024-02-07T17:47:04.676931Z","shell.execute_reply.started":"2024-02-07T17:47:04.575787Z","shell.execute_reply":"2024-02-07T17:47:04.675904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(model, dataloader, optimizer, scheduler, device = device):\n    model.to(device)\n    model.train()\n    train_loss = 0.0\n    scaler = torch.cuda.amp.GradScaler(enabled = use_amp)\n    for X, y in tqdm(dataloader, total = len(dataloader), leave = False):\n        \n        with torch.autocast(device_type=device, dtype=torch.float16, enabled=use_amp):\n            X = X.to(device)\n            y = y.to(device)\n            \n            output = model(X)\n            log_output = torch.nn.functional.log_softmax(output, dim = 1)\n\n            loss = criterion(log_output, y)\n            train_loss += loss.item()\n            \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n#     print(train_loss)\n    scheduler.step()\n    return train_loss / dataloader.batch_size\n\n\ndef valid_epoch(model, dataloader, optimizer, device = device):\n    model.to(device)\n    model.eval()\n    valid_loss = 0.0\n    outputs = []\n    softmax = torch.nn.Softmax(dim = 1)\n    with torch.no_grad():\n        for X, y in tqdm(dataloader, total = len(dataloader), leave = False):\n            X = X.to(device)\n            y = y.to(device)\n\n            output = model(X)\n            log_output = torch.nn.functional.log_softmax(output, dim = 1)\n            softmax_output = softmax(output)\n            outputs.append(output.cpu().numpy())\n\n            loss = criterion(log_output, y)\n            valid_loss += loss.item()\n    \n    outputs = np.concatenate(outputs, axis = 0)\n    return valid_loss / dataloader.batch_size, outputs","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:58:40.544908Z","iopub.execute_input":"2024-02-07T17:58:40.545291Z","iopub.status.idle":"2024-02-07T17:58:40.556806Z","shell.execute_reply.started":"2024-02-07T17:58:40.545263Z","shell.execute_reply":"2024-02-07T17:58:40.555672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_losses = []\nall_oof = []\nall_true = []\n\nfor i, (train_indices, val_indices) in enumerate(splits.split(train, train.target, train.patient_id)):\n    print('*' * 25 + f' Fold {i+1} ' + '*' * 25)\n    \n    train_dataset = HMSData(kaggle_spec, chris_spec, train.iloc[train_indices], 'train')\n    val_dataset = HMSData(kaggle_spec, chris_spec, train.iloc[val_indices], 'valid')\n    \n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = batch_size, shuffle = True)\n    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size = batch_size, shuffle = True)\n    \n    model = Model()\n    optimizer = torch.optim.Adam(model.parameters())\n    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda = lrfn)\n    train_loss = 0.0\n    for epoch in range(EPOCHS):\n        print(f'Running Fold : {i + 1}, Epoch : {epoch + 1}')\n        train_loss += train_epoch(model, train_loader, optimizer, scheduler)\n        \n    print(f'Fold : {i + 1}, Train Loss : {(train_loss / EPOCHS)}')\n    torch.save(model, f'model_fold_{i + 1}.pt')\n    \n    valid_loss, outputs = valid_epoch(model, val_loader, optimizer)\n    print(f'Fold : {i + 1}, Validation Loss : {valid_loss}')\n    \n    all_true.append(train.iloc[val_indices][targets].values)\n#     print(train.iloc[val_indices][targets].values)\n    all_oof.append(outputs)\n    \n    \nall_oof = np.concatenate(all_oof)\nall_true = np.concatenate(all_true)","metadata":{"execution":{"iopub.status.busy":"2024-02-07T17:58:40.991712Z","iopub.execute_input":"2024-02-07T17:58:40.992652Z","iopub.status.idle":"2024-02-07T18:07:17.070758Z","shell.execute_reply.started":"2024-02-07T17:58:40.992608Z","shell.execute_reply":"2024-02-07T18:07:17.069803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}