{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"markdown","source":"## import","metadata":{}},{"cell_type":"code","source":"!pip install flash-attn --no-build-isolation\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T23:22:43.178216Z","iopub.execute_input":"2024-02-11T23:22:43.179009Z","iopub.status.idle":"2024-02-11T23:23:12.087149Z","shell.execute_reply.started":"2024-02-11T23:22:43.178974Z","shell.execute_reply":"2024-02-11T23:23:12.086156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ntrain_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:47.969658Z","iopub.execute_input":"2024-02-11T22:43:47.970451Z","iopub.status.idle":"2024-02-11T22:43:48.505574Z","shell.execute_reply.started":"2024-02-11T22:43:47.970410Z","shell.execute_reply":"2024-02-11T22:43:48.504592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('patient_id').agg(\n{\n    'seizure_vote': 'sum',\n    'lpd_vote' : np.sum,\n    'gpd_vote' : np.sum,\n    'lrda_vote': np.sum,\n    'grda_vote': np.sum,\n    'other_vote': np.sum,\n}\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:48.507207Z","iopub.execute_input":"2024-02-11T22:43:48.507493Z","iopub.status.idle":"2024-02-11T22:43:48.532423Z","shell.execute_reply.started":"2024-02-11T22:43:48.507467Z","shell.execute_reply":"2024-02-11T22:43:48.531544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n_ = plt.hist(np.log10(train_df.groupby('patient_id')['eeg_sub_id'].nunique().values), bins = 100)\nplt.ylabel('Log10 Number of patients')\nplt.xlabel('Log10 Number of samples')\n\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:48.533483Z","iopub.execute_input":"2024-02-11T22:43:48.533744Z","iopub.status.idle":"2024-02-11T22:43:48.965736Z","shell.execute_reply.started":"2024-02-11T22:43:48.533720Z","shell.execute_reply":"2024-02-11T22:43:48.964780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, KFold, train_test_split\n\ntrain_df['sampling_index'] = train_df['spectrogram_id'].astype(str) + \"_\" + train_df['eeg_sub_id'].astype(str)\n\ntrain_set, test_set = train_test_split(train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index())\n\nsampling_df = train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:48.966758Z","iopub.execute_input":"2024-02-11T22:43:48.967045Z","iopub.status.idle":"2024-02-11T22:43:49.724656Z","shell.execute_reply.started":"2024-02-11T22:43:48.967019Z","shell.execute_reply":"2024-02-11T22:43:49.723642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, KFold, train_test_split\n\nsample_size = 55\ncv = KFold(n_splits=5, shuffle=True)\nfor train_idx, val_idx in cv.split(train_set):\n    train_set_slice = train_set.iloc[train_idx]\n    val_set_slice = train_set.iloc[val_idx]\n    train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                            np.unique(\n                                                                                np.random.choice(x, size=sample_size, replace=True)\n                                                                            )\n                                                                        ).values\n                                    )\n    val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n    sampled_train = train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n    sampled_val = train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n    \n    \n    print(\"Sampled_train\", sampled_train.shape)\n    print(\"Sampled_val\",sampled_val.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:49.726973Z","iopub.execute_input":"2024-02-11T22:43:49.727263Z","iopub.status.idle":"2024-02-11T22:43:50.776553Z","shell.execute_reply.started":"2024-02-11T22:43:49.727237Z","shell.execute_reply":"2024-02-11T22:43:50.775672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, KFold, train_test_split\nStratifiedKFold()\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n#             sampling_df = train_df.groupby(['patient_id', 'spectrogram_id']).eeg_id.apply(list).reset_index()\n            self.sampling_df = train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size = 0.1)\n            \n            self.folds = []\n            cv = KFold(n_splits=5, shuffle=True)\n            for train_idx, val_idx in cv.split(self.train_set):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n                self.folds.append((sampled_train, sampled_val))\n                print(sampled_train.head())\n                break\n                \n    \n        else:\n            raise ValueError()\n\n        \n    def validate_model(model):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                model.fit(sampled_train)\n                res = model.predict(sampled_val)\n        else:\n            raise ValueError()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:50.777969Z","iopub.execute_input":"2024-02-11T22:43:50.778252Z","iopub.status.idle":"2024-02-11T22:43:50.790321Z","shell.execute_reply.started":"2024-02-11T22:43:50.778228Z","shell.execute_reply":"2024-02-11T22:43:50.789427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, num_layers, output_dim, dropout=0.1):\n        super(LSTMModel, self).__init__()\n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.num_layers = num_layers\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True, dropout=dropout)\n        \n        # Fully connected output layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    @autocast()  # Apply mixed-precision training\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Initialize cell state with zeros\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Forward propagate LSTM\n        out, _ = self.lstm(x, (h0, c0))\n        \n        # Decode the hidden state of the last time step\n        out = self.fc(out[:, -1, :])\n        return out\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = torch.log_softmax(predicted, dim=-1)\n    target = torch.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = torch.nn.functional.kl_div(predicted, target, reduction='batchmean')\n    \n    return kl_loss\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1)\n            \n            self.folds = []\n            cv = KFold(n_splits=5, shuffle=True)\n            for train_idx, val_idx in cv.split(self.train_set):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                train_dataloader = create_dataloader(sampled_train, batch_size=256, shuffle=True)\n                val_dataloader = create\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:50.791496Z","iopub.execute_input":"2024-02-11T22:43:50.791769Z","iopub.status.idle":"2024-02-11T22:43:52.354514Z","shell.execute_reply.started":"2024-02-11T22:43:50.791745Z","shell.execute_reply":"2024-02-11T22:43:52.353712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh /kaggle/input/hms-harmful-brain-activity-classification/train_eegs/","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:43:52.355601Z","iopub.execute_input":"2024-02-11T22:43:52.356006Z","iopub.status.idle":"2024-02-11T22:44:00.672779Z","shell.execute_reply.started":"2024-02-11T22:43:52.355980Z","shell.execute_reply":"2024-02-11T22:44:00.671651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\ntest_eeg_id = 1628180742\npath = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{test_eeg_id}.parquet\"\ntest_eeg = pl.read_parquet(path, \n                           row_count_offset=0.0 * 200, \n                           n_rows = 10000)","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:00.674466Z","iopub.execute_input":"2024-02-11T22:44:00.674890Z","iopub.status.idle":"2024-02-11T22:44:00.752358Z","shell.execute_reply.started":"2024-02-11T22:44:00.674832Z","shell.execute_reply":"2024-02-11T22:44:00.751430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nimport polars as pl\nfrom tqdm.notebook import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        #todo laplace fix\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\ndef create_dataloader(dataframe, batch_size=32, shuffle=True, num_workers=0):\n    dataset = EEGDataset(dataframe)\n    dataloader = DataLoader(dataset, \n                            batch_size=batch_size, \n                            shuffle=shuffle,\n                            num_workers=num_workers)\n    return dataloader\n\n\n# Example usage:\n# Assuming you have a dataframe named 'train_df' containing the data\ndataloader = create_dataloader(train_df, batch_size=256, shuffle=True)\n\n# Iterate over batches\nfor batch in tqdm(dataloader):\n    eeg_data = batch['eeg_data']\n    labels = batch['labels']\n    # Your training/validation loop here\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:00.753773Z","iopub.execute_input":"2024-02-11T22:44:00.754090Z","iopub.status.idle":"2024-02-11T22:44:06.906725Z","shell.execute_reply.started":"2024-02-11T22:44:00.754064Z","shell.execute_reply":"2024-02-11T22:44:06.905741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = F.softmax(predicted, dim=-1)\n    target = F.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = F.kl_div(predicted.log(), target, reduction='batchmean')\n    \n    return kl_loss","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:06.908037Z","iopub.execute_input":"2024-02-11T22:44:06.908417Z","iopub.status.idle":"2024-02-11T22:44:06.914744Z","shell.execute_reply.started":"2024-02-11T22:44:06.908381Z","shell.execute_reply":"2024-02-11T22:44:06.913858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport polars as pl\nfrom tqdm import tqdm\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import GradScaler, autocast\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\ndef create_dataloader(dataframe, batch_size=32, shuffle=True, num_workers=0):\n    dataset = EEGDataset(dataframe)\n    dataloader = DataLoader(dataset, \n                            batch_size=batch_size, \n                            shuffle=shuffle,\n                            num_workers=num_workers)\n    return dataloader\n\n# Define the LSTM model\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, num_layers, output_dim, dropout=0.1):\n        super(LSTMModel, self).__init__()\n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.num_layers = num_layers\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True, dropout=dropout)\n        \n        # Fully connected output layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    @autocast()  # Apply mixed-precision training\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Initialize cell state with zeros\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Forward propagate LSTM\n        out, _ = self.lstm(x, (h0, c0))\n        \n        # Decode the hidden state of the last time step\n        out = self.fc(out[:, -1, :])\n        return out\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = torch.log_softmax(predicted, dim=-1)\n#     target = torch.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = torch.nn.functional.kl_div(predicted, target, reduction='batchmean')\n    \n    return kl_loss\n\n# # Assuming you have a dataframe named 'train_df' containing the data\n# train_dataloader = create_dataloader(train_df, batch_size=256, shuffle=True)\n\n# # Define model parameters\n# input_dim = 20  # Number of channels\n# output_dim = 6  # Number of output dimensions\n# hidden_dim = 64  # Hidden dimension of the LSTM\n# num_layers = 1  # Number of LSTM layers\n# dropout = 0.1  # Dropout probability\n\n# # Create model instance\n# model = LSTMModel(input_dim, hidden_dim, num_layers, output_dim).cuda()\n\n# # Create optimizer\n# optimizer = AdamW(model.parameters(), lr=1e-3)\n\n# # Enable mixed-precision training\n# scaler = GradScaler()\n\n# # Training loop\n# num_epochs = 10\n\n# for epoch in range(num_epochs):\n#     model.train()\n#     total_loss = 0.0\n    \n#     for batch in tqdm(train_dataloader):\n#         eeg_data = batch['eeg_data'].cuda()\n#         labels = batch['labels'].cuda()\n        \n#         optimizer.zero_grad()\n        \n#         with autocast():\n#             output = model(eeg_data)\n#             loss = kl_divergence_loss(output, labels)\n        \n#         scaler.scale(loss).backward()\n#         scaler.step(optimizer)\n#         scaler.update()\n        \n#         total_loss += loss.item()\n    \n#     avg_loss = total_loss / len(train_dataloader)\n#     print(f\"Epoch {epoch+1}, Average Loss: {avg_loss:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:06.916272Z","iopub.execute_input":"2024-02-11T22:44:06.916621Z","iopub.status.idle":"2024-02-11T22:44:06.935689Z","shell.execute_reply.started":"2024-02-11T22:44:06.916587Z","shell.execute_reply":"2024-02-11T22:44:06.934890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW, Adam\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000).select(pl.all().forward_fill()).select(pl.all().backward_fill())\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW, Adam\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm import tqdm\n\n# Define your Dataset and Model classes here\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55, debug=False, random_seed=4545):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        if debug:\n            self.train_df = self.train_df.head(1000)\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1, random_state=random_seed)\n            \n            self.folds = []\n            cv = KFold(n_splits=nfolds, shuffle=True, random_state=random_seed)\n            for train_idx, val_idx in cv.split(self.train_set, ):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].sample(frac=0.1).reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model, batch_size=256, num_epochs=5, patience=3):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                train_dataloader = create_dataloader(sampled_train, batch_size=batch_size, shuffle=True)\n                val_dataloader = create_dataloader(sampled_val, batch_size=batch_size, shuffle=False)\n                \n                model.cuda()\n                optimizer = Adam(model.parameters(), lr=1e-2)\n                scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=patience, verbose=True)\n                criterion = nn.KLDivLoss(reduction='batchmean')\n                \n                best_val_loss = float('inf')\n                no_improvement = 0\n                \n                for epoch in range(num_epochs):\n                    model.train()\n                    total_loss = 0.0\n                    for batch in tqdm(train_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        optimizer.zero_grad()\n\n                        output = model(eeg_data)\n\n                        loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        loss.backward()\n                        optimizer.step()\n                        \n                        total_loss += loss.item()\n                    \n                    avg_loss = total_loss / len(train_dataloader)\n                    \n                    # Validation loop\n                    model.eval()\n                    total_val_loss = 0.0\n                    for batch in tqdm(val_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        with torch.no_grad():\n                            output = model(eeg_data)\n                            val_loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        total_val_loss += val_loss.item()\n                    \n                    avg_val_loss = total_val_loss / len(val_dataloader)\n                    print(f\"Average Loss: {avg_loss:.4f}\\tEpoch {epoch+1}\\tValidation Loss: {avg_val_loss:.4f}\")\n                    \n                    # Adjust learning rate based on validation loss\n                    scheduler.step(avg_val_loss)\n                    \n                    # Check for early stopping\n                    if avg_val_loss < best_val_loss:\n                        best_val_loss = avg_val_loss\n                        no_improvement = 0\n                    else:\n                        no_improvement += 1\n                        if no_improvement >= patience:\n                            print(f\"No improvement for {patience} epochs. Early stopping.\")\n                            break\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:06.937272Z","iopub.execute_input":"2024-02-11T22:44:06.937594Z","iopub.status.idle":"2024-02-11T22:44:06.967254Z","shell.execute_reply.started":"2024-02-11T22:44:06.937563Z","shell.execute_reply":"2024-02-11T22:44:06.966378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nfrom torch.utils.data import Dataset\nfrom tqdm import trange\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe, memmap_dir='./memmap_dir1', stack_files=True):\n        self.dataframe = dataframe\n        self.memmap_dir = memmap_dir\n        self.stack_files = stack_files\n\n        # Create memmap directory if it does not exist\n        os.makedirs(self.memmap_dir, exist_ok=True)\n\n        # Preprocess the files into memmap\n        self.preprocess_memmap()\n\n    def preprocess_memmap(self):\n\n        for idx in trange(len(self.dataframe), desc=\"Preprocessing Memmap\"):\n            row = self.dataframe.loc[idx]\n            eeg_id = row['eeg_id']\n            path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n            memmap_path = os.path.join(self.memmap_dir, f\"{eeg_id}.npy\")\n\n            # Check if memmap file already exists\n            if not os.path.exists(memmap_path):\n                # Read the Parquet file and preprocess\n                eeg = pl.read_parquet(path, \n                                      row_count_offset=int(row['eeg_label_offset_seconds']) * 200, \n                                      n_rows=10000).select(pl.all().forward_fill()).select(pl.all().backward_fill())\n                eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n                # Check for NaN values\n                if np.isnan(eeg_data).any():\n                    print(f\"Found NaN values in file {eeg_id}. Skipping...\")\n                    continue\n\n                # Stack files if required\n                if self.stack_files:\n                    eeg_data = np.stack(eeg_data)\n\n                # Save as memmap\n                np.save(memmap_path, eeg_data)\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n        eeg_id = row['eeg_id']\n        memmap_path = os.path.join(self.memmap_dir, f\"{eeg_id}.npy\")\n        eeg_data = np.load(memmap_path, mmap_mode='r')\n\n        seizure_vote = row['seizure_vote']\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:06.971192Z","iopub.execute_input":"2024-02-11T22:44:06.971483Z","iopub.status.idle":"2024-02-11T22:44:06.985458Z","shell.execute_reply.started":"2024-02-11T22:44:06.971459Z","shell.execute_reply":"2024-02-11T22:44:06.984578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55, debug=False, random_seed=4545, use_wandb=True):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        if debug:\n            self.train_df = self.train_df.head(1000)\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n        self.use_wandb = use_wandb\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1, random_state=random_seed)\n            \n            self.folds = []\n            cv = KFold(n_splits=nfolds, shuffle=True, random_state=random_seed)\n            for train_idx, val_idx in cv.split(self.train_set, ):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].sample(frac=0.5).reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model, params):\n        batch_size = params.get('batch_size', 256)\n        num_epochs = params.get('num_epochs', 5)\n        patience = params.get('patience', 3)\n        optimizer_params = params.get('optimizer_params', {'lr': 1e-2})\n        scheduler_params = params.get('scheduler_params', {'mode': 'min', 'factor': 0.5, 'patience': patience, 'verbose': True})\n        optimizer_name = params.get('optimizer_name', 'Adam')\n        \n        wandb_group = str(np.random.randint(0, 1000))\n        \n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                \n                if self.use_wandb:\n                    user_secrets = UserSecretsClient()\n                    wandb_key = user_secrets.get_secret(\"Wandb_key\")\n                    os.environ['WANDB_API_KEY'] = wandb_key\n                    wandb.init(project=\"HMS-HBAC\", \n                               entity=\"asimandia\",\n                               group = wandb_group)\n                \n                train_dataloader = create_dataloader(sampled_train, batch_size=batch_size, shuffle=True)\n                val_dataloader = create_dataloader(sampled_val, batch_size=batch_size, shuffle=False)\n                \n                model.cuda()\n                optimizer = Adam(model.parameters(), **optimizer_params) if optimizer_name == 'Adam' else SGD(model.parameters(), **optimizer_params)\n                scheduler = ReduceLROnPlateau(optimizer, **scheduler_params)\n                criterion = nn.KLDivLoss(reduction='batchmean')\n                \n                best_val_loss = float('inf')\n                no_improvement = 0\n                \n                wandb.config.update(params)\n                \n                for epoch in range(num_epochs):\n                    model.train()\n                    total_loss = 0.0\n                    for batch in tqdm(train_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        optimizer.zero_grad()\n\n                        output = model(eeg_data)\n\n                        loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        loss.backward()\n                        optimizer.step()\n                        \n                        total_loss += loss.item()\n                    \n                    avg_loss = total_loss / len(train_dataloader)\n                    \n                    # Validation loop\n                    model.eval()\n                    total_val_loss = 0.0\n                    for batch in tqdm(val_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        with torch.no_grad():\n                            output = model(eeg_data)\n                            val_loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        total_val_loss += val_loss.item()\n                    \n                    avg_val_loss = total_val_loss / len(val_dataloader)\n                    print(f\"Average Loss: {avg_loss:.4f}\\tEpoch {epoch+1}\\tValidation Loss: {avg_val_loss:.4f}\")\n                    \n                    # Log losses to WandB\n                    if self.use_wandb:\n                        wandb.log({\"Train Loss\": avg_loss, \"Validation Loss\": avg_val_loss})\n                    \n                    # Adjust learning rate based on validation loss\n                    scheduler.step(avg_val_loss)\n                    \n                    # Check for early stopping\n                    if avg_val_loss < best_val_loss:\n                        best_val_loss = avg_val_loss\n                        no_improvement = 0\n                    else:\n                        no_improvement += 1\n                        if no_improvement >= patience:\n                            print(f\"No improvement for {patience} epochs. Early stopping.\")\n                            break\n                \n                if self.use_wandb:\n                    wandb.log({\"Best Validation Loss\": best_val_loss})\n                \n                # Calculate score on test set\n                test_loss = self.calculate_test_loss(model, val_dataloader, criterion)\n                wandb.log({\"Test Loss\": test_loss})\n                wandb.finish()\n    \n    def calculate_test_loss(self, model, test_dataloader, criterion):\n        model.eval()\n        total_loss = 0.0\n        for batch in tqdm(test_dataloader):\n            eeg_data = batch['eeg_data'].cuda()\n            labels = batch['labels'].cuda()\n\n            with torch.no_grad():\n                output = model(eeg_data)\n                loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n\n            total_loss += loss.item()\n\n        avg_loss = total_loss / len(test_dataloader)\n        return avg_loss\n","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:06.986765Z","iopub.execute_input":"2024-02-11T22:44:06.987114Z","iopub.status.idle":"2024-02-11T22:44:07.437614Z","shell.execute_reply.started":"2024-02-11T22:44:06.987082Z","shell.execute_reply":"2024-02-11T22:44:07.436834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\nclass ResNet_1D_Block(nn.Module):\n    def __init__(self, in_channels, \n                 out_channels, \n                 kernel_size, \n                 stride, \n                 padding, \n                 downsampling):\n        super(ResNet_1D_Block, self).__init__()\n        self.bn1 = nn.BatchNorm1d(num_features=in_channels)\n        self.relu = nn.ReLU(inplace=False)\n        self.dropout = nn.Dropout(p=0.0, inplace=False)\n        self.conv1 = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n        self.conv2 = nn.Conv1d(in_channels=out_channels, \n                               out_channels=out_channels, \n                               kernel_size=kernel_size,\n                               stride=stride, padding=padding, bias=False)\n        self.maxpool = nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n        self.downsampling = downsampling\n\n    def forward(self, x):\n        identity = x\n#         print(x.shape)\n        out = self.bn1(x)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv1(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.conv2(out)\n\n        out = self.maxpool(out)\n#         print(out.shape)\n\n        identity = self.downsampling(x)\n#         print(identity.shape)\n\n        out += identity\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:07.438720Z","iopub.execute_input":"2024-02-11T22:44:07.439066Z","iopub.status.idle":"2024-02-11T22:44:07.449538Z","shell.execute_reply.started":"2024-02-11T22:44:07.439033Z","shell.execute_reply":"2024-02-11T22:44:07.448661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.Size([1, 1280, 999])\ntorch.Size([1, 64, 499])\ntorch.Size([1, 1280, 499])","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:07.450797Z","iopub.execute_input":"2024-02-11T22:44:07.451131Z","iopub.status.idle":"2024-02-11T22:44:07.466849Z","shell.execute_reply.started":"2024-02-11T22:44:07.451100Z","shell.execute_reply":"2024-02-11T22:44:07.466055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass ChannelwiseCNN1DModel(nn.Module):\n    def __init__(self, \n                 input_dim, \n                 output_dim, \n                 num_filters=64, \n                 kernel_size=3, \n                 dropout=0.1, \n                 seq_length=10000,\n                 max_pool_kernel_size=10,\n                 num_conv_layers=2  # Number of additional convolutional layers\n                ):\n        super(ChannelwiseCNN1DModel, self).__init__()\n        self.channelwise_conv_layers = nn.ModuleList()\n        for _ in range(input_dim):\n            self.channelwise_conv_layers.append(nn.Conv1d(in_channels=1,\n                                                          out_channels=num_filters,\n                                                          kernel_size=kernel_size))\n        self.relu = nn.ReLU()\n        self.maxpool = nn.MaxPool1d(kernel_size=max_pool_kernel_size)\n        self.dropout = nn.Dropout(dropout)\n        \n        self.conv_layers = nn.ModuleList()\n        downsampling = nn.Sequential(nn.MaxPool1d(kernel_size=2, stride=2, padding=0)\n                                    )\n        self.conv_layers.append(ResNet_1D_Block(input_dim * num_filters, \n                                                num_filters * input_dim, \n                                                kernel_size, \n                                               1,\n                                               1, \n                                               downsampling)\n#             nn.Conv1d(in_channels=input_dim * num_filters,\n#                                           out_channels=num_filters,\n#                                           kernel_size=kernel_size)\n                               )\n        \n        for _ in range(num_conv_layers - 1):\n            self.conv_layers.append(ResNet_1D_Block(input_dim * num_filters, \n                                                    num_filters * input_dim, \n                                                    kernel_size, \n                                                    1,\n                                                    1,\n                                                    downsampling)\n                                   )\n        \n        # Adjusting the input size for the fully connected layer\n        self.fc_input_size = self.calculate_fc_input_size(input_dim, num_filters, seq_length, max_pool_kernel_size, num_conv_layers)\n        \n        self.fc1 = nn.Linear(self.fc_input_size, output_dim)\n    \n    def calculate_fc_input_size(self, \n                                input_dim, \n                                num_filters, \n                                seq_length, \n                                max_pool_kernel_size, \n                                num_conv_layers):\n        # Calculate the size of the tensor after passing through convolutions and pooling\n        # This function is used to compute the input size for the fully connected layer\n        x = torch.randn(1, seq_length, 20)  # Create a dummy input tensor\n        batch_size, seq_len, input_channels = x.size()\n        x_list = []\n\n        for i in range(input_channels):  # Loop over input channels\n            x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n            x_channel = self.channelwise_conv_layers[i](x_channel)\n            x_channel = self.relu(x_channel)\n            x_channel = self.maxpool(x_channel)\n            x_channel = self.dropout(x_channel)\n            x_list.append(x_channel)\n        \n        x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n        \n        for conv_layer in self.conv_layers:\n            x = conv_layer(x)\n            x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.dropout(x)\n        x = torch.flatten(x, start_dim=1)  # Flatten along sequence_length dimension\n        return x.size(1)\n    \n    def forward(self, x, debug = False):\n        batch_size, seq_len, input_channels = x.size()\n        x_list = []\n\n        for i in range(input_channels):  # Loop over input channels\n            x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n            x_channel = self.channelwise_conv_layers[i](x_channel)\n            x_channel = self.relu(x_channel)\n            x_channel = self.maxpool(x_channel)\n            x_channel = self.dropout(x_channel)\n            x_list.append(x_channel)\n        \n        x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n        \n        for conv_layer in self.conv_layers:\n            x = conv_layer(x)\n            x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.dropout(x)\n        x = torch.flatten(x, start_dim=1)  # Flatten along sequence_length dimension\n        x = self.fc1(x)\n        return x\n    \n# input_dim = 20  # Number of input channels\n# output_dim = 6  # Number of output dimensions\n# model = ChannelwiseCNN1DModel(input_dim, \n#                               output_dim, \n#                               num_conv_layers=i).cuda()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:17.597578Z","iopub.execute_input":"2024-02-11T22:44:17.598446Z","iopub.status.idle":"2024-02-11T22:44:17.616699Z","shell.execute_reply.started":"2024-02-11T22:44:17.598411Z","shell.execute_reply":"2024-02-11T22:44:17.615680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# for i in [2, 4, 6]:\n#     # Example usage:\n#     params = {\n#         'batch_size' : 180,\n#         'num_epochs' : 100,\n#         'patience' : 3, \n#         'optimizer_params' : {\n#             'lr' : 10e-3,\n#         },\n#         'scheduler_params' : {'mode': 'min', \n#                                   'factor': 0.5, \n#                                   'patience': 3, \n#                                   'verbose': True},\n#         'optimizer_name' : 'Adam',\n#         'num_conv_layers' : i\n        \n#     }\n#     input_dim = 20  # Number of input channels\n#     output_dim = 6  # Number of output dimensions\n#     model = ChannelwiseCNN1DModel(input_dim, \n#                                   output_dim, \n#                                   num_conv_layers=i).cuda()\n#     model = nn.DataParallel(model)\n\n#     # Example usage:\n#     # Create an instance of the ValidationSchema\n#     rand_seed = 4545\n#     validation_schema = ValidationSchema(debug=True, \n#                                          random_seed = rand_seed, \n#                                          nfolds=3)\n#     # Validate the model using the defined schema\n#     validation_schema.validate_model(model, \n#                                      params\n#                                     )","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:18.888305Z","iopub.execute_input":"2024-02-11T22:44:18.888671Z","iopub.status.idle":"2024-02-11T22:44:18.894579Z","shell.execute_reply.started":"2024-02-11T22:44:18.888639Z","shell.execute_reply":"2024-02-11T22:44:18.893545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in [2, 4, 6]:\n#     # Example usage:\n#     params = {\n#         'batch_size' : 180,\n#         'num_epochs' : 100,\n#         'patience' : 3, \n#         'optimizer_params' : {\n#             'lr' : 10e-3,\n#         },\n#         'scheduler_params' : {'mode': 'min', \n#                                   'factor': 0.5, \n#                                   'patience': 3, \n#                                   'verbose': True},\n#         'optimizer_name' : 'Adam',\n#         'num_conv_layers' : i\n        \n#     }","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:08.232217Z","iopub.status.idle":"2024-02-11T22:44:08.232564Z","shell.execute_reply.started":"2024-02-11T22:44:08.232400Z","shell.execute_reply":"2024-02-11T22:44:08.232417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_train_func():\n    wandb.init()\n#     patience = wandb.config.patience\n    num_conv_layers = wandb.config.num_conv_layers\n\n    params = {\n        'batch_size' : 90,\n        'num_epochs' : 100,\n        'patience' : 10, \n        'optimizer_params' : {\n            'lr' : 10e-3,\n        },\n        'scheduler_params' : {'mode': 'min', \n                                  'factor': 0.5, \n                                  'patience': 2, \n                                  'verbose': True},\n        'optimizer_name' : 'Adam',\n        'num_conv_layers' : num_conv_layers\n    }\n    input_dim = 20  # Number of input channels\n    output_dim = 6  # Number of output dimensions\n    model = ChannelwiseCNN1DModel(input_dim, \n                                  output_dim, \n                                  num_conv_layers=i).cuda()\n    model = nn.DataParallel(model)\n\n    # Example usage:\n    # Create an instance of the ValidationSchema\n    rand_seed = 4545\n    validation_schema = ValidationSchema(debug=False, \n                                         random_seed = rand_seed, \n                                         nfolds=3)\n    # Validate the model using the defined schema\n    validation_schema.validate_model(model, \n                                     params\n                                    )","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:23.793394Z","iopub.execute_input":"2024-02-11T22:44:23.794200Z","iopub.status.idle":"2024-02-11T22:44:23.803818Z","shell.execute_reply.started":"2024-02-11T22:44:23.794159Z","shell.execute_reply":"2024-02-11T22:44:23.802927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sweep_configuration = {\n    \"name\": \"my-awesome-sweep-resnet-1d\",\n    \"metric\": {\"name\": \"Best Validation Loss\", \n               \"goal\": \"minimize\"},\n    \"method\": \"grid\",\n    \"parameters\": {\n#         'patience' : {\"values\" : [3, 10]}, \n        'num_conv_layers' : {\"values\" : [1, 2, 4]}\n    },\n}\n\nuser_secrets = UserSecretsClient()\nwandb_key = user_secrets.get_secret(\"Wandb_key\")\nos.environ['WANDB_API_KEY'] = wandb_key\n\nwandb.login(key=wandb_key)\n\nsweep_id = wandb.sweep(sweep_configuration, \n                       project=\"HMS-HBAC\", \n                       entity=\"asimandia\",\n                      )\n\n# run the sweep\nwandb.agent(sweep_id, function=my_train_func, count=30)","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:24.243369Z","iopub.execute_input":"2024-02-11T22:44:24.243718Z","iopub.status.idle":"2024-02-11T22:44:49.129093Z","shell.execute_reply.started":"2024-02-11T22:44:24.243688Z","shell.execute_reply":"2024-02-11T22:44:49.128130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1 + 1","metadata":{"execution":{"iopub.status.busy":"2024-02-11T22:44:08.237817Z","iopub.status.idle":"2024-02-11T22:44:08.238184Z","shell.execute_reply.started":"2024-02-11T22:44:08.238023Z","shell.execute_reply":"2024-02-11T22:44:08.238040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}