{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"markdown","source":"## import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ntrain_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:02.908229Z","iopub.execute_input":"2024-02-10T20:10:02.908690Z","iopub.status.idle":"2024-02-10T20:10:03.389076Z","shell.execute_reply.started":"2024-02-10T20:10:02.908659Z","shell.execute_reply":"2024-02-10T20:10:03.388249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('patient_id').agg(\n{\n    'seizure_vote': 'sum',\n    'lpd_vote' : np.sum,\n    'gpd_vote' : np.sum,\n    'lrda_vote': np.sum,\n    'grda_vote': np.sum,\n    'other_vote': np.sum,\n}\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:03.392322Z","iopub.execute_input":"2024-02-10T20:10:03.392545Z","iopub.status.idle":"2024-02-10T20:10:03.422596Z","shell.execute_reply.started":"2024-02-10T20:10:03.392523Z","shell.execute_reply":"2024-02-10T20:10:03.421891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n_ = plt.hist(np.log10(train_df.groupby('patient_id')['eeg_sub_id'].nunique().values), bins = 100)\nplt.ylabel('Log10 Number of patients')\nplt.xlabel('Log10 Number of samples')\n\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:03.423505Z","iopub.execute_input":"2024-02-10T20:10:03.423709Z","iopub.status.idle":"2024-02-10T20:10:03.754443Z","shell.execute_reply.started":"2024-02-10T20:10:03.423688Z","shell.execute_reply":"2024-02-10T20:10:03.753631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, KFold, train_test_split\n\ntrain_df['sampling_index'] = train_df['spectrogram_id'].astype(str) + \"_\" + train_df['eeg_sub_id'].astype(str)\n\ntrain_set, test_set = train_test_split(train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index())\n\nsampling_df = train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:03.756452Z","iopub.execute_input":"2024-02-10T20:10:03.756663Z","iopub.status.idle":"2024-02-10T20:10:05.096890Z","shell.execute_reply.started":"2024-02-10T20:10:03.756641Z","shell.execute_reply":"2024-02-10T20:10:05.096240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 55\ncv = KFold(n_splits=5, shuffle=True)\nfor train_idx, val_idx in cv.split(train_set):\n    train_set_slice = train_set.iloc[train_idx]\n    val_set_slice = train_set.iloc[val_idx]\n    train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                            np.unique(\n                                                                                np.random.choice(x, size=sample_size, replace=True)\n                                                                            )\n                                                                        ).values\n                                    )\n    val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n    sampled_train = train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n    sampled_val = train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n    \n    \n    print(\"Sampled_train\", sampled_train.shape)\n    print(\"Sampled_val\",sampled_val.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:05.098071Z","iopub.execute_input":"2024-02-10T20:10:05.098792Z","iopub.status.idle":"2024-02-10T20:10:05.899042Z","shell.execute_reply.started":"2024-02-10T20:10:05.098761Z","shell.execute_reply":"2024-02-10T20:10:05.898244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, KFold, train_test_split\nStratifiedKFold()\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n#             sampling_df = train_df.groupby(['patient_id', 'spectrogram_id']).eeg_id.apply(list).reset_index()\n            self.sampling_df = train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size = 0.1)\n            \n            self.folds = []\n            cv = KFold(n_splits=5, shuffle=True)\n            for train_idx, val_idx in cv.split(self.train_set):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n                self.folds.append((sampled_train, sampled_val))\n                print(sampled_train.head())\n                break\n                \n    \n        else:\n            raise ValueError()\n\n        \n    def validate_model(model):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                model.fit(sampled_train)\n                res = model.predict(sampled_val)\n        else:\n            raise ValueError()","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:05.900289Z","iopub.execute_input":"2024-02-10T20:10:05.900755Z","iopub.status.idle":"2024-02-10T20:10:05.912018Z","shell.execute_reply.started":"2024-02-10T20:10:05.900725Z","shell.execute_reply":"2024-02-10T20:10:05.911395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, num_layers, output_dim, dropout=0.1):\n        super(LSTMModel, self).__init__()\n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.num_layers = num_layers\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True, dropout=dropout)\n        \n        # Fully connected output layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    @autocast()  # Apply mixed-precision training\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Initialize cell state with zeros\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Forward propagate LSTM\n        out, _ = self.lstm(x, (h0, c0))\n        \n        # Decode the hidden state of the last time step\n        out = self.fc(out[:, -1, :])\n        return out\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = torch.log_softmax(predicted, dim=-1)\n    target = torch.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = torch.nn.functional.kl_div(predicted, target, reduction='batchmean')\n    \n    return kl_loss\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1)\n            \n            self.folds = []\n            cv = KFold(n_splits=5, shuffle=True)\n            for train_idx, val_idx in cv.split(self.train_set):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                train_dataloader = create_dataloader(sampled_train, batch_size=256, shuffle=True)\n                val_dataloader = create\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:05.912740Z","iopub.execute_input":"2024-02-10T20:10:05.913008Z","iopub.status.idle":"2024-02-10T20:10:08.195798Z","shell.execute_reply.started":"2024-02-10T20:10:05.912980Z","shell.execute_reply":"2024-02-10T20:10:08.195026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh /kaggle/input/hms-harmful-brain-activity-classification/train_eegs/","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:08.196936Z","iopub.execute_input":"2024-02-10T20:10:08.197481Z","iopub.status.idle":"2024-02-10T20:10:31.964482Z","shell.execute_reply.started":"2024-02-10T20:10:08.197458Z","shell.execute_reply":"2024-02-10T20:10:31.963847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\ntest_eeg_id = 1628180742\npath = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{test_eeg_id}.parquet\"\ntest_eeg = pl.read_parquet(path, \n                           row_count_offset=0.0 * 200, \n                           n_rows = 10000)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:31.965515Z","iopub.execute_input":"2024-02-10T20:10:31.965817Z","iopub.status.idle":"2024-02-10T20:10:32.233020Z","shell.execute_reply.started":"2024-02-10T20:10:31.965782Z","shell.execute_reply":"2024-02-10T20:10:32.232389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nimport polars as pl\nfrom tqdm.notebook import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        #todo laplace fix\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\ndef create_dataloader(dataframe, batch_size=32, shuffle=True, num_workers=0):\n    dataset = EEGDataset(dataframe)\n    dataloader = DataLoader(dataset, \n                            batch_size=batch_size, \n                            shuffle=shuffle,\n                            num_workers=num_workers)\n    return dataloader\n\n\n# Example usage:\n# Assuming you have a dataframe named 'train_df' containing the data\ndataloader = create_dataloader(train_df, batch_size=256, shuffle=True)\n\n# Iterate over batches\nfor batch in tqdm(dataloader):\n    eeg_data = batch['eeg_data']\n    labels = batch['labels']\n    # Your training/validation loop here\n    break","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:32.235953Z","iopub.execute_input":"2024-02-10T20:10:32.236334Z","iopub.status.idle":"2024-02-10T20:10:39.533041Z","shell.execute_reply.started":"2024-02-10T20:10:32.236310Z","shell.execute_reply":"2024-02-10T20:10:39.532221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = F.softmax(predicted, dim=-1)\n    target = F.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = F.kl_div(predicted.log(), target, reduction='batchmean')\n    \n    return kl_loss","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:39.533886Z","iopub.execute_input":"2024-02-10T20:10:39.534122Z","iopub.status.idle":"2024-02-10T20:10:39.538660Z","shell.execute_reply.started":"2024-02-10T20:10:39.534099Z","shell.execute_reply":"2024-02-10T20:10:39.537962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport polars as pl\nfrom tqdm import tqdm\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import GradScaler, autocast\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000)\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\ndef create_dataloader(dataframe, batch_size=32, shuffle=True, num_workers=0):\n    dataset = EEGDataset(dataframe)\n    dataloader = DataLoader(dataset, \n                            batch_size=batch_size, \n                            shuffle=shuffle,\n                            num_workers=num_workers)\n    return dataloader\n\n# Define the LSTM model\nclass LSTMModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim, num_layers, output_dim, dropout=0.1):\n        super(LSTMModel, self).__init__()\n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.num_layers = num_layers\n        \n        # LSTM layer\n        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True, dropout=dropout)\n        \n        # Fully connected output layer\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    @autocast()  # Apply mixed-precision training\n    def forward(self, x):\n        # Initialize hidden state with zeros\n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Initialize cell state with zeros\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).cuda()\n        \n        # Forward propagate LSTM\n        out, _ = self.lstm(x, (h0, c0))\n        \n        # Decode the hidden state of the last time step\n        out = self.fc(out[:, -1, :])\n        return out\n\ndef kl_divergence_loss(predicted, target):\n    # Ensure both predicted and target have probabilities summing up to 1\n    predicted = torch.log_softmax(predicted, dim=-1)\n#     target = torch.softmax(target, dim=-1)\n    \n    # Compute KL divergence\n    kl_loss = torch.nn.functional.kl_div(predicted, target, reduction='batchmean')\n    \n    return kl_loss\n\n# # Assuming you have a dataframe named 'train_df' containing the data\n# train_dataloader = create_dataloader(train_df, batch_size=256, shuffle=True)\n\n# # Define model parameters\n# input_dim = 20  # Number of channels\n# output_dim = 6  # Number of output dimensions\n# hidden_dim = 64  # Hidden dimension of the LSTM\n# num_layers = 1  # Number of LSTM layers\n# dropout = 0.1  # Dropout probability\n\n# # Create model instance\n# model = LSTMModel(input_dim, hidden_dim, num_layers, output_dim).cuda()\n\n# # Create optimizer\n# optimizer = AdamW(model.parameters(), lr=1e-3)\n\n# # Enable mixed-precision training\n# scaler = GradScaler()\n\n# # Training loop\n# num_epochs = 10\n\n# for epoch in range(num_epochs):\n#     model.train()\n#     total_loss = 0.0\n    \n#     for batch in tqdm(train_dataloader):\n#         eeg_data = batch['eeg_data'].cuda()\n#         labels = batch['labels'].cuda()\n        \n#         optimizer.zero_grad()\n        \n#         with autocast():\n#             output = model(eeg_data)\n#             loss = kl_divergence_loss(output, labels)\n        \n#         scaler.scale(loss).backward()\n#         scaler.step(optimizer)\n#         scaler.update()\n        \n#         total_loss += loss.item()\n    \n#     avg_loss = total_loss / len(train_dataloader)\n#     print(f\"Epoch {epoch+1}, Average Loss: {avg_loss:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:39.539459Z","iopub.execute_input":"2024-02-10T20:10:39.540146Z","iopub.status.idle":"2024-02-10T20:10:39.554035Z","shell.execute_reply.started":"2024-02-10T20:10:39.540123Z","shell.execute_reply":"2024-02-10T20:10:39.553294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW, Adam\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm import tqdm\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n\n        # Extract relevant information from the dataframe row\n        eeg_id = row['eeg_id']\n        spectrogram_id = row['spectrogram_id']\n        seizure_vote = row['seizure_vote']  # Assuming these columns exist in your dataframe\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        # Construct the path to the parquet file\n        path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n\n        # Read the parquet file\n        eeg = pl.read_parquet(path, \n                              row_count_offset=float(row['eeg_label_offset_seconds']) * 200, \n                              n_rows=10000).select(pl.all().forward_fill()).select(pl.all().backward_fill())\n\n        # Assuming you have some specific way of processing the data, modify the following line accordingly\n        eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n        # Assuming you have labels associated with the EEG data\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data': eeg_data, 'labels': labels}\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import AdamW, Adam\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nimport numpy as np\nfrom tqdm import tqdm\n\n# Define your Dataset and Model classes here\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55, debug=False, random_seed=4545):\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        if debug:\n            self.train_df = self.train_df.head(1000)\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1, random_state=random_seed)\n            \n            self.folds = []\n            cv = KFold(n_splits=nfolds, shuffle=True, random_state=random_seed)\n            for train_idx, val_idx in cv.split(self.train_set, ):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].sample(frac=0.1).reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model, batch_size=256, num_epochs=5, patience=3):\n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                train_dataloader = create_dataloader(sampled_train, batch_size=batch_size, shuffle=True)\n                val_dataloader = create_dataloader(sampled_val, batch_size=batch_size, shuffle=False)\n                \n                model.cuda()\n                optimizer = Adam(model.parameters(), lr=1e-2)\n                scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=patience, verbose=True)\n                criterion = nn.KLDivLoss(reduction='batchmean')\n                \n                best_val_loss = float('inf')\n                no_improvement = 0\n                \n                for epoch in range(num_epochs):\n                    model.train()\n                    total_loss = 0.0\n                    for batch in tqdm(train_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        optimizer.zero_grad()\n\n                        output = model(eeg_data)\n\n                        loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        loss.backward()\n                        optimizer.step()\n                        \n                        total_loss += loss.item()\n                    \n                    avg_loss = total_loss / len(train_dataloader)\n                    \n                    # Validation loop\n                    model.eval()\n                    total_val_loss = 0.0\n                    for batch in tqdm(val_dataloader):\n                        eeg_data = batch['eeg_data'].cuda()\n                        labels = batch['labels'].cuda()\n                        \n                        with torch.no_grad():\n                            output = model(eeg_data)\n                            val_loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        total_val_loss += val_loss.item()\n                    \n                    avg_val_loss = total_val_loss / len(val_dataloader)\n                    print(f\"Average Loss: {avg_loss:.4f}\\tEpoch {epoch+1}\\tValidation Loss: {avg_val_loss:.4f}\")\n                    \n                    # Adjust learning rate based on validation loss\n                    scheduler.step(avg_val_loss)\n                    \n                    # Check for early stopping\n                    if avg_val_loss < best_val_loss:\n                        best_val_loss = avg_val_loss\n                        no_improvement = 0\n                    else:\n                        no_improvement += 1\n                        if no_improvement >= patience:\n                            print(f\"No improvement for {patience} epochs. Early stopping.\")\n                            break\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:39.554770Z","iopub.execute_input":"2024-02-10T20:10:39.554963Z","iopub.status.idle":"2024-02-10T20:10:39.575639Z","shell.execute_reply.started":"2024-02-10T20:10:39.554944Z","shell.execute_reply":"2024-02-10T20:10:39.574857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Вот тут вот поменял внузу датасета, чтобы он выдавал нам только кусуки","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nfrom torch.utils.data import Dataset\nfrom tqdm import trange\n\nclass EEGDataset(Dataset):\n    def __init__(self, dataframe, memmap_dir='./memmap_dir1', stack_files=True):\n        self.dataframe = dataframe\n        self.memmap_dir = memmap_dir\n        self.stack_files = stack_files\n\n        # Create memmap directory if it does not exist\n        os.makedirs(self.memmap_dir, exist_ok=True)\n\n        # Preprocess the files into memmap\n        self.preprocess_memmap()\n\n    def preprocess_memmap(self):\n\n        for idx in trange(len(self.dataframe), desc=\"Preprocessing Memmap\"):\n            row = self.dataframe.loc[idx]\n            eeg_id = row['eeg_id']\n            path = f\"/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/{eeg_id}.parquet\"\n            memmap_path = os.path.join(self.memmap_dir, f\"{eeg_id}.npy\")\n\n            # Check if memmap file already exists\n            if not os.path.exists(memmap_path):\n                # Read the Parquet file and preprocess\n                eeg = pl.read_parquet(path, \n                                      row_count_offset=int(row['eeg_label_offset_seconds']) * 200, \n                                      n_rows=10000).select(pl.all().forward_fill()).select(pl.all().backward_fill())\n                eeg_data = eeg.to_numpy(use_pyarrow=False)\n\n                # Check for NaN values\n                if np.isnan(eeg_data).any():\n                    print(f\"Found NaN values in file {eeg_id}. Skipping...\")\n                    continue\n\n                # Stack files if required\n                if self.stack_files:\n                    eeg_data = np.stack(eeg_data)\n\n                # Save as memmap\n                np.save(memmap_path, eeg_data)\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.loc[idx]\n        eeg_id = row['eeg_id']\n        memmap_path = os.path.join(self.memmap_dir, f\"{eeg_id}.npy\")\n        eeg_data = np.load(memmap_path, mmap_mode='r')\n\n        seizure_vote = row['seizure_vote']\n        lpd_vote = row['lpd_vote']\n        gpd_vote = row['gpd_vote']\n        lrda_vote = row['lrda_vote']\n        grda_vote = row['grda_vote']\n        other_vote = row['other_vote']\n\n        labels = torch.tensor([seizure_vote, lpd_vote, gpd_vote, lrda_vote, grda_vote, other_vote], \n                              dtype=torch.float32)\n        # Normalize the labels\n        labels = labels / labels.sum()\n        return {'eeg_data_10': eeg_data[:200*10, :], 'eeg_data_20': eeg_data[200*10: 200*20, :],\n                'eeg_data_30': eeg_data[200*20: 200*30, :], 'eeg_data_40': eeg_data[200*30: 200*40, :],\n                'eeg_data_50': eeg_data[200*40: ,:], 'labels': labels}\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:39.576620Z","iopub.execute_input":"2024-02-10T20:10:39.576930Z","iopub.status.idle":"2024-02-10T20:10:39.588259Z","shell.execute_reply.started":"2024-02-10T20:10:39.576876Z","shell.execute_reply":"2024-02-10T20:10:39.587688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Добавил изменения батча из датасета и чтобы девайс передавать можно было и не дебагать на видеокарте. Местами добавил if для использования wandb, где его не было, потому что у меня ключ не добавлется почему-то и код падает","metadata":{}},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nclass ValidationSchema:\n    def __init__(self, nfolds=5, stratified=True, mode='patient', sample_size=55, debug=False, random_seed=4545, use_wandb=True, device='cuda'):\n        self.device = device\n        self.nfolds = nfolds\n        self.stratified = True\n        self.mode = mode\n        self.models = []\n        self.train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n        if debug:\n            self.train_df = self.train_df.head(1000)\n        self.train_df['sampling_index'] = self.train_df['spectrogram_id'].astype(str) + \"_\" + self.train_df['eeg_sub_id'].astype(str)\n        self.use_wandb = use_wandb\n\n        if self.mode == 'patient':\n            self.sampling_df = self.train_df.groupby(['patient_id']).sampling_index.apply(list).reset_index()\n            self.train_set, self.test_set = train_test_split(self.sampling_df, test_size=0.1, random_state=random_seed)\n            \n            self.folds = []\n            cv = KFold(n_splits=nfolds, shuffle=True, random_state=random_seed)\n            for train_idx, val_idx in cv.split(self.train_set, ):\n                train_set_slice = self.train_set.iloc[train_idx]\n                val_set_slice = self.train_set.iloc[val_idx]\n                train_eegs_subs = np.concatenate(train_set_slice['sampling_index'].apply(lambda x: \n                                                                                        np.unique(\n                                                                                            np.random.choice(x, size=sample_size, replace=True)\n                                                                                        )\n                                                                                    ).values\n                                                )\n                val_eegs_subs = np.concatenate(val_set_slice['sampling_index'].values)\n                sampled_train = self.train_df.set_index('sampling_index').loc[train_eegs_subs].reset_index()\n                sampled_val = self.train_df.set_index('sampling_index').loc[val_eegs_subs].sample(frac=0.1).reset_index()\n                self.folds.append((sampled_train, sampled_val))\n                print(f\"Train fold size {sampled_train.shape} || Test fold size {sampled_val.shape}\")\n    \n    def validate_model(self, model, params):\n        batch_size = params.get('batch_size', 256)\n        num_epochs = params.get('num_epochs', 5)\n        patience = params.get('patience', 3)\n        optimizer_params = params.get('optimizer_params', {'lr': 1e-2})\n        scheduler_params = params.get('scheduler_params', {'mode': 'min', 'factor': 0.5, 'patience': patience, 'verbose': True})\n        optimizer_name = params.get('optimizer_name', 'Adam')\n        \n        wandb_group = str(np.random.randint(0, 1000))\n        \n        if self.mode == 'patient':\n            for sampled_train, sampled_val in self.folds:\n                \n                if self.use_wandb:\n                    user_secrets = UserSecretsClient()\n                    wandb_key = user_secrets.get_secret(\"Wandb_key\")\n                    os.environ['WANDB_API_KEY'] = wandb_key\n                    wandb.init(project=\"HMS-HBAC\", \n                               entity=\"asimandia\",\n                               group = wandb_group)\n                \n                train_dataloader = create_dataloader(sampled_train, batch_size=batch_size, shuffle=True)\n                val_dataloader = create_dataloader(sampled_val, batch_size=batch_size, shuffle=False)\n                \n                model = model.to(device)\n                optimizer = Adam(model.parameters(), **optimizer_params) if optimizer_name == 'Adam' else SGD(model.parameters(), **optimizer_params)\n                scheduler = ReduceLROnPlateau(optimizer, **scheduler_params)\n                criterion = nn.KLDivLoss(reduction='batchmean')\n                \n                best_val_loss = float('inf')\n                no_improvement = 0\n                \n                if self.use_wandb:\n                    wandb.config.update(params)\n                \n                for epoch in range(num_epochs):\n                    model.train()\n                    total_loss = 0.0\n                    for batch in tqdm(train_dataloader):\n                        eeg_data_10, eeg_data_20, eeg_data_30, eeg_data_40, eeg_data_50 = batch['eeg_data_10'].to(self.device), batch['eeg_data_20'].to(self.device), batch['eeg_data_30'].to(self.device), batch['eeg_data_40'].to(self.device), batch['eeg_data_50'].to(self.device)\n                        labels = batch['labels'].to(self.device)\n                        \n                        optimizer.zero_grad()\n\n                        output = model(eeg_data_10, eeg_data_20, eeg_data_30, eeg_data_40, eeg_data_50)\n\n                        loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        loss.backward()\n                        optimizer.step()\n                        \n                        total_loss += loss.item()\n                    \n                    avg_loss = total_loss / len(train_dataloader)\n                    \n                    # Validation loop\n                    model.eval()\n                    total_val_loss = 0.0\n                    for batch in tqdm(val_dataloader):\n                        eeg_data_10, eeg_data_20, eeg_data_30, eeg_data_40, eeg_data_50 = batch['eeg_data_10'].to(self.device), batch['eeg_data_20'].to(self.device), batch['eeg_data_30'].to(self.device), batch['eeg_data_40'].to(self.device), batch['eeg_data_50'].to(self.device)\n                        labels = batch['labels'].to(self.device)\n                        \n                        with torch.no_grad():\n                            output = model(eeg_data_10, eeg_data_20, eeg_data_30, eeg_data_40, eeg_data_50)\n                            val_loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n                        \n                        total_val_loss += val_loss.item()\n                    \n                    avg_val_loss = total_val_loss / len(val_dataloader)\n                    print(f\"Average Loss: {avg_loss:.4f}\\tEpoch {epoch+1}\\tValidation Loss: {avg_val_loss:.4f}\")\n                    \n                    # Log losses to WandB\n                    if self.use_wandb:\n                        wandb.log({\"Train Loss\": avg_loss, \"Validation Loss\": avg_val_loss})\n                    \n                    # Adjust learning rate based on validation loss\n                    scheduler.step(avg_val_loss)\n                    \n                    # Check for early stopping\n                    if avg_val_loss < best_val_loss:\n                        best_val_loss = avg_val_loss\n                        no_improvement = 0\n                    else:\n                        no_improvement += 1\n                        if no_improvement >= patience:\n                            print(f\"No improvement for {patience} epochs. Early stopping.\")\n                            break\n                \n                if self.use_wandb:\n                    wandb.log({\"Best Validation Loss\": best_val_loss})\n                \n                # Calculate score on test set\n                test_loss = self.calculate_test_loss(model, val_dataloader, criterion)\n                if self.use_wandb:\n                    wandb.log({\"Test Loss\": test_loss})\n    \n    def calculate_test_loss(self, model, test_dataloader, criterion):\n        model.eval()\n        total_loss = 0.0\n        for batch in tqdm(test_dataloader):\n            eeg_data_10, eeg_data_20, eeg_data_30, eeg_data_40, eeg_data_50 = batch['eeg_data_10'].to(self.device), batch['eeg_data_20'].to(self.device), batch['eeg_data_30'].to(self.device), batch['eeg_data_40'].to(self.device), batch['eeg_data_50'].to(self.device)\n            labels = batch['labels'].to(self.device)\n\n            with torch.no_grad():\n                output = model(eeg_data)\n                loss = criterion(F.log_softmax(output, dim=1), F.softmax(labels, dim=1))\n\n            total_loss += loss.item()\n\n        avg_loss = total_loss / len(test_dataloader)\n        return avg_loss\n","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:39.589836Z","iopub.execute_input":"2024-02-10T20:10:39.590041Z","iopub.status.idle":"2024-02-10T20:10:40.242644Z","shell.execute_reply.started":"2024-02-10T20:10:39.590021Z","shell.execute_reply":"2024-02-10T20:10:40.242009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass ChannelwiseCNN1DModel(nn.Module):\n    def __init__(self, \n                 input_dim, \n                 output_dim, \n                 num_filters=64, \n                 kernel_size=3, \n                 dropout=0.1, \n                 seq_length=10000,\n                 max_pool_kernel_size=10,\n                 num_conv_layers=2  # Number of additional convolutional layers\n                ):\n        super(ChannelwiseCNN1DModel, self).__init__()\n        self.channelwise_conv_layers = nn.ModuleList()\n        for _ in range(input_dim):\n            self.channelwise_conv_layers.append(nn.Conv1d(in_channels=1,\n                                                          out_channels=num_filters,\n                                                          kernel_size=kernel_size))\n        self.relu = nn.ReLU()\n        self.maxpool = nn.MaxPool1d(kernel_size=max_pool_kernel_size)\n        self.dropout = nn.Dropout(dropout)\n        \n        self.conv_layers = nn.ModuleList()\n        self.conv_layers.append(nn.Conv1d(in_channels=input_dim * num_filters,\n                                          out_channels=num_filters,\n                                          kernel_size=kernel_size))\n        for _ in range(num_conv_layers - 1):\n            self.conv_layers.append(nn.Conv1d(in_channels=num_filters,\n                                              out_channels=num_filters,\n                                              kernel_size=kernel_size))\n        \n        # Adjusting the input size for the fully connected layer\n        self.fc_input_size = self.calculate_fc_input_size(input_dim, num_filters, seq_length, max_pool_kernel_size, num_conv_layers)\n        \n        self.fc1 = nn.Linear(self.fc_input_size, output_dim)\n    \n    def calculate_fc_input_size(self, \n                                input_dim, \n                                num_filters, \n                                seq_length, \n                                max_pool_kernel_size, \n                                num_conv_layers):\n        # Calculate the size of the tensor after passing through convolutions and pooling\n        # This function is used to compute the input size for the fully connected layer\n        x = torch.randn(1, seq_length, 20)  # Create a dummy input tensor\n        batch_size, seq_len, input_channels = x.size()\n        x_list = []\n\n        for i in range(input_channels):  # Loop over input channels\n            x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n            x_channel = self.channelwise_conv_layers[i](x_channel)\n            x_channel = self.relu(x_channel)\n            x_channel = self.maxpool(x_channel)\n            x_channel = self.dropout(x_channel)\n            x_list.append(x_channel)\n        \n        x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n        \n        for conv_layer in self.conv_layers:\n            x = conv_layer(x)\n            x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.dropout(x)\n        x = torch.flatten(x, start_dim=1)  # Flatten along sequence_length dimension\n#         x = self.fc1(x)\n        return x.size(1)\n    \n    def forward(self, x, debug = False):\n        batch_size, seq_len, input_channels = x.size()\n        x_list = []\n\n        for i in range(input_channels):  # Loop over input channels\n            x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n            x_channel = self.channelwise_conv_layers[i](x_channel)\n            x_channel = self.relu(x_channel)\n            x_channel = self.maxpool(x_channel)\n            x_channel = self.dropout(x_channel)\n            x_list.append(x_channel)\n        \n        x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n        \n        for conv_layer in self.conv_layers:\n            x = conv_layer(x)\n            x = self.relu(x)\n        x = self.maxpool(x)\n        x = self.dropout(x)\n        x = torch.flatten(x, start_dim=1)  # Flatten along sequence_length dimension\n        x = self.fc1(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:40.243709Z","iopub.execute_input":"2024-02-10T20:10:40.244079Z","iopub.status.idle":"2024-02-10T20:10:40.256521Z","shell.execute_reply.started":"2024-02-10T20:10:40.244057Z","shell.execute_reply":"2024-02-10T20:10:40.255771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Написал слой внимания","metadata":{}},{"cell_type":"code","source":"class AttentionLayer(nn.Module):\n    def __init__(self, enc_size, dec_size, hid_size, activ=torch.tanh):\n        \"\"\" A layer that computes additive attention response and weights \"\"\"\n        super().__init__()\n        self.enc_size = enc_size\n        self.dec_size = dec_size\n        self.hid_size = hid_size\n        self.activ = activ       \n\n        self.fc_enc = nn.Linear(enc_size, hid_size, bias=False)\n        self.fc_dec = nn.Linear(dec_size, hid_size, bias=False)\n        self.fc_att = nn.Linear(hid_size, 1, bias=False)\n\n\n    def forward(self, enc, dec):\n\n        logits_enc = self.fc_enc(enc)\n        logits_dec = self.fc_dec(dec)\n        logits = self.fc_att(self.activ(logits_enc + logits_dec.unsqueeze(1))).squeeze(-1)\n        \n        probs = nn.functional.softmax(logits, dim=-1).squeeze(1)\n        \n        attn = torch.einsum('ijkh,ijk->ikh', enc, probs)\n        return attn","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:10:40.257422Z","iopub.execute_input":"2024-02-10T20:10:40.257688Z","iopub.status.idle":"2024-02-10T20:10:40.270831Z","shell.execute_reply.started":"2024-02-10T20:10:40.257662Z","shell.execute_reply":"2024-02-10T20:10:40.269737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Встроил слой внимания так, как я это вижу","metadata":{}},{"cell_type":"code","source":"class ChannelwiseCNN1DModelWithAtt(nn.Module):\n    def __init__(self, \n                 input_dim, \n                 output_dim, \n                 num_filters=64, \n                 kernel_size=3, \n                 dropout=0.1, \n                 seq_length=10000,\n                 max_pool_kernel_size=10,\n                 num_conv_layers=2  # Number of additional convolutional layers\n                ):\n        super(ChannelwiseCNN1DModelWithAtt, self).__init__()\n        \n        self.channelwise_conv_layers = nn.ModuleList()\n        \n        for _ in range(input_dim):\n            self.channelwise_conv_layers.append(nn.Conv1d(in_channels=1,\n                                                          out_channels=num_filters,\n                                                          kernel_size=kernel_size))\n        \n        self.relu = nn.ReLU()\n        self.maxpool = nn.MaxPool1d(kernel_size=max_pool_kernel_size)\n        self.dropout = nn.Dropout(dropout)\n        \n        self.conv_layers = nn.ModuleList()\n        self.conv_layers.append(nn.Conv1d(in_channels=input_dim * num_filters,\n                                          out_channels=num_filters,\n                                          kernel_size=kernel_size))\n        for _ in range(num_conv_layers - 1):\n            self.conv_layers.append(nn.Conv1d(in_channels=num_filters,\n                                              out_channels=num_filters,\n                                              kernel_size=kernel_size))\n            \n            \n        self.att_input_size, x, y = self.calculate_att_input_size(input_dim, num_filters, seq_length//5, max_pool_kernel_size, num_conv_layers)\n        self.att = AttentionLayer(self.att_input_size, self.att_input_size, self.att_input_size*2)\n        \n        # Adjusting the input size for the fully connected layer\n        self.fc_input_size = self.calculate_fc_input_size(x, y, input_dim, num_filters, seq_length//5, max_pool_kernel_size, num_conv_layers)\n        \n        self.fc1 = nn.Linear(self.fc_input_size, output_dim)\n    \n    def calculate_att_input_size(self,\n                                input_dim,\n                                num_filters,\n                                seq_length,\n                                max_pool_kernel_size,\n                                num_conv_layers):\n        \n        x = torch.randn(1, seq_length, 20)  # Create a dummy input tensor\n        y = torch.randn(1, seq_length, 20)  # Create dummy to use in attention\n        batch_size, seq_len, input_channels = x.size()\n        x_list = []\n        y_list = []\n\n        for i in range(input_channels):  # Loop over input channels\n            x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n            y_channel = y[:, :, i].unsqueeze(1)\n            x_channel = self.channelwise_conv_layers[i](x_channel)\n            y_channel = self.channelwise_conv_layers[i](y_channel)\n            x_channel = self.relu(x_channel)\n            y_channel = self.relu(y_channel)\n            x_channel = self.maxpool(x_channel)\n            y_channel = self.maxpool(y_channel)\n            x_channel = self.dropout(x_channel)\n            y_channel = self.dropout(y_channel)\n            x_list.append(x_channel)\n            y_list.append(y_channel)\n        x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n        \n        x = x.unsqueeze(1).expand(-1, 4, -1, -1)\n        y = torch.cat(y_list, dim=1)   \n        \n        return x.shape[-1], x, y\n    \n    def calculate_fc_input_size(self,\n                                x, y, \n                                input_dim, \n                                num_filters, \n                                seq_length, \n                                max_pool_kernel_size, \n                                num_conv_layers):\n        \n        x = self.att(x, y)\n        y = y + x\n        \n        for conv_layer in self.conv_layers:\n            y = conv_layer(y)\n            y = self.relu(y)\n        y = self.maxpool(y)\n        y = self.dropout(y)\n        y = torch.flatten(y, start_dim=1)\n        \n#         x = self.fc1(x)\n        return y.size(1)\n    \n    def forward(self, x1, x2, x3, x4, x5, debug = False):\n        x_data = [x1, x2, x3, x4, x5]\n        x_res = []\n        batch_size, seq_len, input_channels = x1.size()\n        for x in x_data:\n            x_list = []\n            for i in range(input_channels):  # Loop over input channels\n                x_channel = x[:, :, i].unsqueeze(1)  # Get a single channel and add channel dimension\n                x_channel = self.channelwise_conv_layers[i](x_channel)\n                x_channel = self.relu(x_channel)\n                x_channel = self.maxpool(x_channel)\n                x_channel = self.dropout(x_channel)\n                x_list.append(x_channel)\n            \n            x = torch.cat(x_list, dim=1)  # Concatenate along the channel dimension\n            x_res.append(x.unsqueeze(1))\n        \n        x3 = x_res.pop(2).squeeze(1)\n        x_res = torch.cat(x_res, dim=1)\n        \n        x3 = self.att(x_res, x3)\n            \n        for conv_layer in self.conv_layers:\n            x3 = conv_layer(x3)\n            x3 = self.relu(x3)\n        x3 = self.maxpool(x3)\n        x3 = self.dropout(x3)\n        x3 = torch.flatten(x3, start_dim=1)  # Flatten along sequence_length dimension\n        x3 = self.fc1(x3)\n        return x3","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:11:14.045988Z","iopub.execute_input":"2024-02-10T20:11:14.046330Z","iopub.status.idle":"2024-02-10T20:11:14.064235Z","shell.execute_reply.started":"2024-02-10T20:11:14.046299Z","shell.execute_reply":"2024-02-10T20:11:14.063662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Следующие 5 ячеек — я просто тестил, что все работает","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nfor i in [3]:\n    # Example usage:\n    params = {\n        'batch_size' : 180,\n        'num_epochs' : 100,\n        'patience' : 3, \n        'optimizer_params' : {\n            'lr' : 10e-3,\n        },\n        'scheduler_params' : {'mode': 'min', \n                                  'factor': 0.5, \n                                  'patience': 3, \n                                  'verbose': True},\n        'optimizer_name' : 'Adam',\n        'num_conv_layers' : i\n        \n    }\n    input_dim = 20  # Number of input channels\n    output_dim = 6  # Number of output dimensions\n    model = ChannelwiseCNN1DModelWithAtt(input_dim, \n                                  output_dim, \n                                  num_conv_layers=i).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:11:33.744845Z","iopub.execute_input":"2024-02-10T20:11:33.745175Z","iopub.status.idle":"2024-02-10T20:11:33.819740Z","shell.execute_reply.started":"2024-02-10T20:11:33.745144Z","shell.execute_reply":"2024-02-10T20:11:33.818913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:11:35.256076Z","iopub.execute_input":"2024-02-10T20:11:35.257066Z","iopub.status.idle":"2024-02-10T20:11:35.262954Z","shell.execute_reply.started":"2024-02-10T20:11:35.257040Z","shell.execute_reply":"2024-02-10T20:11:35.262084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_seed = 4545\nvalidation_schema = ValidationSchema(debug=False, \n                                     random_seed = rand_seed, \n                                     nfolds=3, use_wandb=False, device=device)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:11:37.092123Z","iopub.execute_input":"2024-02-10T20:11:37.092610Z","iopub.status.idle":"2024-02-10T20:11:37.996105Z","shell.execute_reply.started":"2024-02-10T20:11:37.092586Z","shell.execute_reply":"2024-02-10T20:11:37.995362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = EEGDataset(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:11:39.540001Z","iopub.execute_input":"2024-02-10T20:11:39.540343Z","iopub.status.idle":"2024-02-10T20:21:19.317137Z","shell.execute_reply.started":"2024-02-10T20:11:39.540316Z","shell.execute_reply":"2024-02-10T20:21:19.315265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x1, x2, x3, x4, x5 = torch.tensor(dataset[0]['eeg_data_10']), torch.tensor(dataset[0]['eeg_data_10']), torch.tensor(dataset[0]['eeg_data_10']), torch.tensor(dataset[0]['eeg_data_10']), torch.tensor(dataset[0]['eeg_data_10'])\nmodel(x1.unsqueeze(0), x2.unsqueeze(0), x3.unsqueeze(0), x4.unsqueeze(0), x5.unsqueeze(0))","metadata":{"execution":{"iopub.status.busy":"2024-02-10T20:21:19.320641Z","iopub.execute_input":"2024-02-10T20:21:19.321179Z","iopub.status.idle":"2024-02-10T20:21:19.470451Z","shell.execute_reply.started":"2024-02-10T20:21:19.321141Z","shell.execute_reply":"2024-02-10T20:21:19.469667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nfor i in [2, 4, 6]:\n    # Example usage:\n    params = {\n        'batch_size' : 180,\n        'num_epochs' : 100,\n        'patience' : 3, \n        'optimizer_params' : {\n            'lr' : 10e-3,\n        },\n        'scheduler_params' : {'mode': 'min', \n                                  'factor': 0.5, \n                                  'patience': 3, \n                                  'verbose': True},\n        'optimizer_name' : 'Adam',\n        'num_conv_layers' : i\n        \n    }\n    input_dim = 20  # Number of input channels\n    output_dim = 6  # Number of output dimensions\n    model = ChannelwiseCNN1DModelWithAtt(input_dim, \n                                  output_dim, \n                                  num_conv_layers=i).to(device)\n    model = nn.DataParallel(model)\n\n    # Example usage:\n    # Create an instance of the ValidationSchema\n    rand_seed = 4545\n    validation_schema = ValidationSchema(debug=False, \n                                         random_seed = rand_seed, \n                                         nfolds=3, use_wandb=False, device=device)\n    # Validate the model using the defined schema\n    validation_schema.validate_model(model, \n                                     params\n                                    )","metadata":{"execution":{"iopub.status.busy":"2024-02-10T19:26:06.483370Z","iopub.execute_input":"2024-02-10T19:26:06.483876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}