{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install torch_optimizer > _","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:20.524878Z","iopub.execute_input":"2024-01-12T07:38:20.525226Z","iopub.status.idle":"2024-01-12T07:38:34.110415Z","shell.execute_reply.started":"2024-01-12T07:38:20.525201Z","shell.execute_reply":"2024-01-12T07:38:34.109240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport pytorch_lightning as pl\nfrom torch_optimizer import AdaBound\nfrom torch.nn import functional as F\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-12T07:38:34.112417Z","iopub.execute_input":"2024-01-12T07:38:34.112760Z","iopub.status.idle":"2024-01-12T07:38:41.169360Z","shell.execute_reply.started":"2024-01-12T07:38:34.112731Z","shell.execute_reply":"2024-01-12T07:38:41.168334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = Path(\"/kaggle/input/hms-harmful-brain-activity-classification\")\n\nEEG_SAMPLING_TIME = 50  #second\nEEG_SAMPLING_RATE = 200 #Hz\nEEG_DURATION = EEG_SAMPLING_RATE * EEG_SAMPLING_TIME\n\nSPECTROGRAM_TIME = 10   #minute\n\nN_CLASS = 6\nCHANNEL = 20\nBATCH_SIZE = 128","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.170874Z","iopub.execute_input":"2024-01-12T07:38:41.171843Z","iopub.status.idle":"2024-01-12T07:38:41.177524Z","shell.execute_reply.started":"2024-01-12T07:38:41.171802Z","shell.execute_reply":"2024-01-12T07:38:41.176514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)  # 🎲 Set seed for NumPy\n    torch.manual_seed(seed)  # 🚀 Set seed for PyTorch on CPU\n    torch.cuda.manual_seed(seed)  # 🚀 Set seed for PyTorch on GPU\n    \n    # ⚙️ When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    # 🔏 Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n# 🌱 Set seed using the configured seed value\nset_seed()","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.180282Z","iopub.execute_input":"2024-01-12T07:38:41.180960Z","iopub.status.idle":"2024-01-12T07:38:41.193151Z","shell.execute_reply.started":"2024-01-12T07:38:41.180926Z","shell.execute_reply":"2024-01-12T07:38:41.192298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HMSDataset(Dataset):\n    def __init__(self, df, is_traindir=True, transform=None,):\n        self.df = df\n        self.is_traindir = is_traindir\n        self.transform = transform\n        \n    def _get_train_eeg(self, q:dict) -> pd.DataFrame:\n        parquet_df = pd.read_parquet(BASE_DIR/f\"train_eegs/{q['eeg_id']}.parquet\")\n        eeg_start_index = int(EEG_SAMPLING_RATE * q[\"eeg_label_offset_seconds\"])\n        return parquet_df.iloc[eeg_start_index:eeg_start_index+EEG_DURATION]\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        q_dict = dict(self.df.iloc[idx])\n        if self.is_traindir:\n            eeg_df = self._get_train_eeg(q_dict)\n        else:\n            eeg_df = pd.read_parquet(BASE_DIR/f\"test_eegs/{q_dict['eeg_id']}.parquet\")\n        eeg_df = eeg_df.fillna(0)\n        eeg = eeg_df.values.T\n        if self.transform:\n            eeg = self.transform(eeg)\n        label_count = self.df.iloc[idx].filter(like=\"vote\")\n        label = label_count/label_count.sum()\n        eeg = torch.from_numpy(eeg)\n        label = torch.from_numpy(label.astype(np.float16).values)\n        return eeg, label\n","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.194354Z","iopub.execute_input":"2024-01-12T07:38:41.195022Z","iopub.status.idle":"2024-01-12T07:38:41.205052Z","shell.execute_reply.started":"2024-01-12T07:38:41.194986Z","shell.execute_reply":"2024-01-12T07:38:41.204086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleNet(nn.Module):\n    def __init__(self):\n        super(SimpleNet,self).__init__()\n        self.conv1 = nn.Conv1d(CHANNEL, 8,kernel_size=3, stride=1)\n        self.bn1 = nn.BatchNorm1d(8)\n        self.relu = nn.ReLU()\n        self.maxpool = nn.MaxPool1d(kernel_size=3, stride=2)\n        self.conv2 = nn.Conv1d(8, 16,kernel_size=3, stride=1)\n        self.bn2 = nn.BatchNorm1d(16)\n        self.conv3 = nn.Conv1d(16,64,kernel_size=3, stride=1)\n        self.gap = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Linear(64, N_CLASS)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.conv3(x)\n        x = self.gap(x)\n        x = x.view(x.size(0),-1)\n        x = self.fc(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.206218Z","iopub.execute_input":"2024-01-12T07:38:41.206543Z","iopub.status.idle":"2024-01-12T07:38:41.219289Z","shell.execute_reply.started":"2024-01-12T07:38:41.206516Z","shell.execute_reply":"2024-01-12T07:38:41.218272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BrainClassificationDM(pl.LightningDataModule):\n    def __init__(\n        self,\n        df,\n        data_transforms,\n    ):\n        super().__init__()\n        self.df = df\n        self.data_transforms = data_transforms\n\n    def prepare_data(self):\n        pass\n\n    @property\n    def num_classes(self) -> int:\n        return N_CLASS\n\n    def setup(self, stage=None):\n        patient_ids = self.df['patient_id'].unique()\n        train_patient_ids, valid_patient_ids = train_test_split(patient_ids, test_size=0.2, shuffle=True, random_state=42)\n        train = self.df[self.df['patient_id'].isin(train_patient_ids)]\n        valid = self.df[self.df['patient_id'].isin(valid_patient_ids)]\n        self.train_dataset = HMSDataset(train,is_traindir=True,transform=self.data_transforms[\"train\"])\n        print(f\"training dataset: {len(self.train_dataset)}\")\n        self.valid_dataset = HMSDataset(valid, is_traindir=True, transform=self.data_transforms[\"valid\"])\n        print(f\"validation dataset: {len(self.valid_dataset)}\")\n\n    def train_dataloader(self):\n        return DataLoader(\n            self.train_dataset, \n            batch_size=BATCH_SIZE, \n            num_workers=os.cpu_count(),  # 🔍 Number of workers for data loading\n            shuffle=True,   # 🔄 Do not shuffle the data for train set\n            pin_memory=True  # 🚀 Pin the memory for faster GPU data transfer\n        )\n\n    def val_dataloader(self):\n        return DataLoader(\n            self.valid_dataset, \n            batch_size=BATCH_SIZE, \n            num_workers=os.cpu_count(),  # 🔍 Number of workers for data loading\n            shuffle=False,   # 🔄 Do not shuffle the data for train set\n            pin_memory=True  # 🚀 Pin the memory for faster GPU data transfer\n        )\n\n    def test_dataloader(self):\n        pass\n\n    \ntrain_df = pd.read_csv(BASE_DIR/\"train.csv\")\ndata_transforms = {\n    \"train\": None,\n    \"valid\": None\n}\ndm = BrainClassificationDM(\n    df=train_df,\n    data_transforms=data_transforms,\n)\ndm.setup()","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.220685Z","iopub.execute_input":"2024-01-12T07:38:41.221717Z","iopub.status.idle":"2024-01-12T07:38:41.676160Z","shell.execute_reply.started":"2024-01-12T07:38:41.221680Z","shell.execute_reply":"2024-01-12T07:38:41.675247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LitCancerSubtype(pl.LightningModule):\n\n    def __init__(self, net, lr: float = 1e-4):\n        super().__init__()\n        self.net = net\n        self.arch = \"baseline\"\n        self.num_classes = N_CLASS\n        self.learn_rate = lr\n\n    def forward(self, x):\n        y = self.net(x)\n        return y\n\n    def compute_loss(self, y_hat, y):\n        return F.cross_entropy(y_hat, y.to(y_hat.dtype))\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        #print(f\"{lbs=} ?= {y_hat=}\")\n        loss = self.compute_loss(y_hat, y)\n        #print(f\"{y=} ?= {y_hat=} -> {loss=}\")\n        self.log(\"train_loss\", loss, logger=True, prog_bar=True)\n        #print(f\"{lb=} ?= {y_hat=} -> {self.train_accuracy(y_hat, lbs)}\")\n#         self.log(\"train_acc\", self.train_accuracy(y_hat, y), logger=True, prog_bar=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.compute_loss(y_hat, y)\n        self.log(\"valid_loss\", loss, logger=True, prog_bar=True)\n\n    def configure_optimizers(self):\n        optimizer = AdaBound(self.parameters(), lr=self.learn_rate)\n        #optimizer = RAdam(self.parameters(), lr=self.learn_rate)\n        #optimizer = torch.optim.AdamW(self.parameters(), lr=self.learn_rate)\n        #optimizer = Lion(self.parameters(), lr=self.learn_rate, weight_decay=1e-2)\n        #optimizer = Adan(self.parameters(), lr=self.learn_rate * 10, betas=(0.02, 0.08, 0.01), weight_decay=0.02)\n        \n        #scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n        #    optimizer, T_max=self.trainer.max_epochs, eta_min=1e-6, verbose=True)\n        scheduler = torch.optim.lr_scheduler.CyclicLR(\n          optimizer, base_lr=self.learn_rate, max_lr=self.learn_rate * 5,\n          step_size_up=5, cycle_momentum=False, mode=\"triangular2\", verbose=True)\n        #scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        #    optimizer, max_lr=self.learn_rate * 5, steps_per_epoch=1, epochs=self.trainer.max_epochs)\n        return [optimizer], [scheduler]\n\n\nnet = SimpleNet()\nmodel = LitCancerSubtype(net=net, lr=1e-4)","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.677697Z","iopub.execute_input":"2024-01-12T07:38:41.677991Z","iopub.status.idle":"2024-01-12T07:38:41.769929Z","shell.execute_reply.started":"2024-01-12T07:38:41.677966Z","shell.execute_reply":"2024-01-12T07:38:41.769136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logger = pl.loggers.CSVLogger(save_dir='logs/', name=model.arch)\nnb_epochs = 10\nearly_stop_metric = \"valid_loss\"\nearly_stop_mode = \"min\"\n# ==============================\n\ntrainer = pl.Trainer(\n    accelerator=\"cuda\",\n    # devices=2,\n    # fast_dev_run=True,\n    # callbacks=[swa],\n    # precision=16,\n    #val_check_interval=0.5,\n    logger=logger,\n    min_epochs=3,\n    max_epochs=nb_epochs,\n    accumulate_grad_batches=1,\n    callbacks=[\n        EarlyStopping(\n            monitor=early_stop_metric,\n            min_delta=0.00,\n            patience=3,\n            verbose=False,\n            mode=early_stop_mode,\n            )\n        ]\n)\n\n\ntrainer.fit(model=model, datamodule=dm)\ntrainer.save_checkpoint(\"outputs/seq_classification_model.pt\")","metadata":{"execution":{"iopub.status.busy":"2024-01-12T07:38:41.771009Z","iopub.execute_input":"2024-01-12T07:38:41.771250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = pd.read_csv(f'{trainer.logger.log_dir}/metrics.csv')\ndel metrics[\"step\"]\nmetrics.set_index(\"epoch\", inplace=True)\n# display(metrics.dropna(axis=1, how=\"all\").head())\ng = sns.relplot(data=metrics, kind=\"line\")\nplt.gcf().set_size_inches(12, 4)\n# plt.gca().set_yscale('log')\nplt.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}