{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7437371,"sourceType":"datasetVersion","datasetId":4328630},{"sourceId":7437573,"sourceType":"datasetVersion","datasetId":4328761},{"sourceId":161531087,"sourceType":"kernelVersion"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":102.485133,"end_time":"2024-01-16T12:14:21.10455","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-16T12:12:38.619417","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# !!!***!!! STATUS\n\n* Code trains but loss doesnt seem to go down over 5 or so epochs\n* Takes 13 mins/epoch. I started code to preprocess EEG so it runs faster when grabbing with dataloader. The preprocessing (which should only need to happen once) is running overnight. If doesn't work, look at what others have done in discussion. Otherwise modify dataloader to use it and see if speeds up training time.\n* Once convinced training works, if going to take too long, start with Regis' weights \n","metadata":{"execution":{"iopub.status.busy":"2024-03-24T05:37:22.36374Z","iopub.execute_input":"2024-03-24T05:37:22.364026Z","iopub.status.idle":"2024-03-24T05:37:22.368842Z","shell.execute_reply.started":"2024-03-24T05:37:22.364Z","shell.execute_reply":"2024-03-24T05:37:22.367788Z"}}},{"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt, gc\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom scipy.signal import butter, filtfilt, lfilter\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom scipy.signal import freqz\nfrom sklearn.model_selection import train_test_split\n\nfrom datetime import datetime\n\nimport torch as t\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom tqdm import tqdm\n\nprint(\"done\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.693827,"end_time":"2024-01-16T12:12:42.606147","exception":false,"start_time":"2024-01-16T12:12:41.91232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-25T20:42:51.688651Z","iopub.execute_input":"2024-03-25T20:42:51.689064Z","iopub.status.idle":"2024-03-25T20:42:51.698701Z","shell.execute_reply.started":"2024-03-25T20:42:51.689032Z","shell.execute_reply":"2024-03-25T20:42:51.697113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"import gc \ndef GC():\n    gc.collect()\n    t.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T20:20:34.128773Z","iopub.execute_input":"2024-03-25T20:20:34.129534Z","iopub.status.idle":"2024-03-25T20:20:34.134177Z","shell.execute_reply.started":"2024-03-25T20:20:34.129498Z","shell.execute_reply":"2024-03-25T20:20:34.132950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"#Define Paths\nBASE_PATH = \"/kaggle/input/hms-harmful-brain-activity-classification\"\nTRAIN_EEG_PATH = f'{BASE_PATH}/train_eegs'\n\nWORKING_BASE_PATH = \"/kaggle/working\"\nPROCESSED_EEG_PATH = f'{WORKING_BASE_PATH}/train_eegs_processed'\nos.makedirs(PROCESSED_EEG_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T20:20:34.135605Z","iopub.execute_input":"2024-03-25T20:20:34.136012Z","iopub.status.idle":"2024-03-25T20:20:34.147100Z","shell.execute_reply.started":"2024-03-25T20:20:34.135979Z","shell.execute_reply":"2024-03-25T20:20:34.146149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get CSV data\nLABELED_DATA_df = pd.read_csv(f'{BASE_PATH}/train.csv')\nUNLABELED_DATA_df = pd.read_csv(f'{BASE_PATH}/test.csv')\n\n# Define target columns(votes for each activity pattern)\nLABEL_COLS = LABELED_DATA_df.columns[-6:]\n\n#---- CELL OUTPUT ------------:\nprint(f\"\\nLABEL COLUMNS:\")\nfor col in LABEL_COLS:\n    print(\"  -\",col)\n    \nprint('\\n\\nLABELED DATA:', LABELED_DATA_df.shape, '\\n' )\ndisplay(LABELED_DATA_df.head() )\n\nprint('\\n\\nUNLABELED DATA:', UNLABELED_DATA_df.shape, '\\n' )\ndisplay(UNLABELED_DATA_df.head() )","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.693827,"end_time":"2024-01-16T12:12:42.606147","exception":false,"start_time":"2024-01-16T12:12:41.91232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-25T20:20:34.148485Z","iopub.execute_input":"2024-03-25T20:20:34.148859Z","iopub.status.idle":"2024-03-25T20:20:34.505546Z","shell.execute_reply.started":"2024-03-25T20:20:34.148826Z","shell.execute_reply":"2024-03-25T20:20:34.504510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_signals(signals):\n    \"\"\"\n    Normalize EEG signals.\n    \n    Parameters:\n    - signals: NumPy array of EEG signals, shape (signal_length, n_channels).\n    \n    Returns:\n    - Normalized signals.\n    \"\"\"\n    # maybe add clipping\n    mean = signals.mean(axis=0, keepdims=True)\n    std = signals.std(axis=0, keepdims=True)\n    return (signals - mean) / std","metadata":{"execution":{"iopub.status.busy":"2024-03-25T20:20:34.507564Z","iopub.execute_input":"2024-03-25T20:20:34.508101Z","iopub.status.idle":"2024-03-25T20:20:34.515167Z","shell.execute_reply.started":"2024-03-25T20:20:34.508068Z","shell.execute_reply":"2024-03-25T20:20:34.513997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def butter_bandpass_filter(data, lowcut =1 , highcut = 40, fs=200, order=6):\n    b, a = butter(order, [lowcut / (0.5 * fs), highcut / (0.5 * fs)], btype='band')\n    y = lfilter(b, a, data)\n    return y","metadata":{"execution":{"iopub.status.busy":"2024-03-25T20:46:29.733603Z","iopub.execute_input":"2024-03-25T20:46:29.734059Z","iopub.status.idle":"2024-03-25T20:46:29.741861Z","shell.execute_reply.started":"2024-03-25T20:46:29.734028Z","shell.execute_reply":"2024-03-25T20:46:29.740354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Find all EEG snippets and save as individual files for faster loading by dataloader\n\nrun_preprocessing = True\n\ndef preprocess_and_save(df, eeg_path, output_path):\n    old_parq_path = ''\n    for idx, row in df.iterrows():\n        eeg_id = row['eeg_id']\n        processed_path = os.path.join(output_path, f'{eeg_id}_processed.npy')\n        if ~os.path.exists(processed_path): #create process eeg file if does not already exist\n            new_parq_path = f'{eeg_path}/{eeg_id}.parquet'\n            if new_parq_path != old_parq_path: #load new parquet file if current eeg is not draw from last opened parquet file\n                eeg_df = pd.read_parquet(new_parq_path)\n                old_parq_path = new_parq_path\n\n            # Crop and clean the data\n            start_time_second = row['eeg_label_offset_seconds']\n            dur_s = 50\n            samp_rate = 200\n            offset_dp = int(start_time_second * samp_rate)\n            dur_dp = dur_s * samp_rate\n            snippet_df = eeg_df.iloc[offset_dp:offset_dp+dur_dp]\n            snippet_df = snippet_df.ffill(axis=0).fillna(0)\n            snippet_np = normalize_signals(snippet_df.values)\n            snippet_np = butter_bandpass_filter(snippet_np)                      \n            assert(snippet_df.shape[0] == dur_dp)\n            \n            #Save\n            np.save(processed_path, snippet_np)\n            plt.plot(snippet_df.values[:1000,4])\n            plt.show()\n            plt.plot(snippet_np[:1000,4])\n            plt.show()\n            break\n        if idx%1000 <1:\n            print(f\"{idx} of {len(df)} processed\")\n\nif run_preprocessing:\n    preprocess_and_save(LABELED_DATA_df, TRAIN_EEG_PATH, PROCESSED_EEG_PATH)\n    output = \"EEG data preprocessed to extract relevant intervals for each dp\"\nelse:\n    output = \"preprocessing turned off (presumably because preprocessed files already saved)\"\n\n#---- CELL OUTPUT ------------:\nprint(output)\n      \n      ","metadata":{"execution":{"iopub.status.busy":"2024-03-25T20:46:32.630451Z","iopub.execute_input":"2024-03-25T20:46:32.630855Z","iopub.status.idle":"2024-03-25T20:46:33.157420Z","shell.execute_reply.started":"2024-03-25T20:46:32.630824Z","shell.execute_reply":"2024-03-25T20:46:33.156151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Datasets and Dataloaders","metadata":{}},{"cell_type":"code","source":"# Create Dataset Class\n\nclass TrainOrValDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx, label_cols = LABEL_COLS, eeg_path = TRAIN_EEG_PATH):    \n        \n        row_series = self.df.iloc[idx]\n        \n        #GET FEATURE DATA (EEG channels)\n        #Read in EEG data\n        eeg_id = row_series['eeg_id']\n        \n        #THIS WAS ALL MOVED INTO PREPROCESSING, DELETE WHEN READY\n#         parq_path = f'{eeg_path}/{eeg_id}.parquet'\n#         eeg_df = pd.read_parquet(parq_path)\n#         #Crop to relevant portion\n#         start_time_second = row_series['eeg_label_offset_seconds']\n#         dur_s = 50\n#         samp_rate = 200\n#         offset_dp = int(start_time_second * samp_rate)\n#         dur_dp =  dur_s * samp_rate\n#         eeg_df = eeg_df.iloc[offset_dp:offset_dp+dur_dp]\n#         #Clean up nulls\n#         eeg_df = eeg_df.ffill(axis=0) #fill nulls with last nonnull\n#         eeg_df = eeg_df.fillna(0) #fill remaining nulls -- ie if any at start of trace -- with 0)\n#         #Get eeg channels of interest\n        \n        #Load EEG clip\n        processed_eeg_path = os.path.join(PROCESSED_EEG_PATH, f'{eeg_id}_processed.npy')\n        eeg_nparray = np.load(processed_eeg_path)\n\n        \n        \n#         eeg_df = pd.DataFrame(eeg_nparray)\n#         eeg_positions = ['Fp1', 'F3', 'C3', 'P3', 'F7', 'T3', 'T5', 'O1', 'Fz', 'Cz', 'Pz', 'Fp2', 'F4', 'C4', 'P4', 'F8', 'T4', 'T6', 'O2', 'EKG']\n#         assert len(eeg_positions) == len(eeg_df.columns)\n#         eeg_df.columns = eeg_positions        \n#       #Convert to tensor\n        features = eeg_df[eeg_positions]\n        \n        #GET LABEL DATA expert votes)\n        label = row_series[label_cols].values.astype(np.float64)\n        label = label/np.sum(label) #convert to pct of total votes\n      \n        return t.tensor(eeg_nparray, dtype=t.float32), t.tensor(label,dtype=t.float64)","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:26.314072Z","iopub.execute_input":"2024-03-24T15:23:26.314542Z","iopub.status.idle":"2024-03-24T15:23:26.325024Z","shell.execute_reply.started":"2024-03-24T15:23:26.314509Z","shell.execute_reply":"2024-03-24T15:23:26.323415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create datasets (train, val, test)\n\nTRAIN_df, VAL_df = train_test_split(LABELED_DATA_df, test_size = .1, random_state = 1)\n\nTRAIN_ds = TrainOrValDataset(TRAIN_df)\nVAL_ds = TrainOrValDataset(VAL_df)\nTEST_ds = TrainOrValDataset(UNLABELED_DATA_df)\n\n#---- CELL OUTPUT ------------:\nprint(f\"Train size: {len(TRAIN_ds)}\")\nprint(f\"Val size: {len(VAL_ds)}\")\nprint(f\"Test size: {len(TEST_ds)}*\")\nprint(\"\"\"\n* NOTE: test size is 1 because this is a code submission competition.\nThey provide one example that can be used to setup submission code to take actual\ntest data, which we don't get to see in advance\"\"\")","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:29.388043Z","iopub.execute_input":"2024-03-24T15:23:29.389136Z","iopub.status.idle":"2024-03-24T15:23:29.431471Z","shell.execute_reply.started":"2024-03-24T15:23:29.38909Z","shell.execute_reply":"2024-03-24T15:23:29.430658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Dataloaders\nfrom torch.utils.data import DataLoader\n\nBATCH_SZ = 64\nN_WORKERS = 1\n\nTRAIN_dl = DataLoader(TRAIN_ds, batch_size=BATCH_SZ, shuffle=True, num_workers=N_WORKERS)\nVAL_dl = DataLoader(VAL_ds, batch_size=BATCH_SZ, shuffle=True, num_workers=N_WORKERS)\nTEST_dl = DataLoader(TEST_ds, batch_size=BATCH_SZ, shuffle=True, num_workers=N_WORKERS)\n\n#---- CELL OUTPUT ------------:\nprint(\"data loaders created\")","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:33.641161Z","iopub.execute_input":"2024-03-24T15:23:33.641604Z","iopub.status.idle":"2024-03-24T15:23:33.649465Z","shell.execute_reply.started":"2024-03-24T15:23:33.64155Z","shell.execute_reply":"2024-03-24T15:23:33.648182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Model -- Separated GRU","metadata":{}},{"cell_type":"code","source":"#Establish device\n\nDEVICE = 'cuda' if t.cuda.is_available() else 'cpu'\nprint(DEVICE)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:36.902031Z","iopub.execute_input":"2024-03-24T15:23:36.902431Z","iopub.status.idle":"2024-03-24T15:23:36.908337Z","shell.execute_reply.started":"2024-03-24T15:23:36.902398Z","shell.execute_reply":"2024-03-24T15:23:36.907463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define model class\n\nclass ConvBlock(nn.Module):\n    def __init__(self, d_in, d_out, kernel_size, drop):\n        super().__init__()\n        self.model = nn.Sequential(\n            nn.Conv1d(d_in, d_out, kernel_size=kernel_size, padding='same', stride=1),\n            nn.ReLU(),\n            nn.Dropout(drop),\n            nn.Conv1d(d_out, d_out, kernel_size=kernel_size, padding='same', stride=1),\n            nn.ReLU(),\n            nn.Dropout(drop),\n            nn.Conv1d(d_out, d_out, kernel_size=kernel_size, padding='same', stride=1),\n            nn.ReLU(),\n            nn.Dropout(drop),\n            nn.MaxPool1d(kernel_size=2, stride=2, padding=0), # reduce sequence size by 2\n        )\n    def forward(self, x):\n        return self.model(x)\n    \nclass GRU(nn.Module):\n    def __init__(self, in_channels=5, gru_hidden_size=128, drop=0.1):\n        super().__init__()\n        self.group_ids = [\n            [0, 1, 2, 3, 7],\n            [0, 4, 5, 6, 7],\n            [11, 12, 13, 14, 18],\n            [11, 15, 16, 17, 18],\n            # [8, 9, 10, 19] # TODO: try with leftovers?\n            ]\n        self.d_split = len(self.group_ids)\n        self.pre_out = in_channels * 4\n        self.gru_hidden_size = gru_hidden_size\n        \n        self.pre_process = nn.Sequential(\n            nn.BatchNorm1d(in_channels, momentum=None),\n            # use conv1d as a denoiser\n            # block 1\n            ConvBlock(in_channels, in_channels * 2, kernel_size=3, drop=drop),\n            # nn.BatchNorm1d(in_channels * 2, momentum=None),\n            \n            # block 2\n            ConvBlock(in_channels * 2, in_channels * 4, kernel_size=5, drop=drop),\n            # nn.BatchNorm1d(self.pre_out, momentum=None),\n\n            # block 3\n            ConvBlock(in_channels * 4, in_channels * 4, kernel_size=7, drop=drop),\n            # nn.BatchNorm1d(self.pre_out, momentum=None),\n        )\n        \n        # TODO: add a learnable first state for GRU or check what is the default\n        self.gru = nn.GRU(self.pre_out, self.gru_hidden_size, num_layers=1, batch_first=True, bidirectional=True)\n\n        self.post_gru = nn.Sequential(\n            nn.Linear(self.gru_hidden_size * 2, self.gru_hidden_size),\n            nn.ReLU(),\n            nn.Linear(self.gru_hidden_size, self.gru_hidden_size),\n        )\n\n        self.head = nn.Sequential(\n            nn.Linear(self.gru_hidden_size * self.d_split, self.gru_hidden_size * 2),\n            nn.ReLU(),\n            nn.Dropout(drop),\n            nn.Linear(self.gru_hidden_size * 2, 6)\n        )\n\n    def forward(self, x):\n        # separate the input into 4 splits (LP, LL, RP, RR)\n        splits = [x[:, :, group] for group in self.group_ids]\n\n        # stack and then reshape to combine group and batch dimensions\n        group_batch, seq, channel = splits[0].shape\n        x = t.stack(splits, dim=0)\n        x = x.reshape(group_batch * len(self.group_ids), seq, channel)\n\n        # pre_process steps\n        x = x.permute(0, 2, 1)  # permute to match the pre_process expected input shape\n        x = self.pre_process(x)\n        x = x.permute(0, 2, 1)  # permute back to original\n\n        # GRU processing\n        x, _ = self.gru(x)\n        x = x[:, -1, :]\n\n        # MLP post GRU\n        x = self.post_gru(x)\n\n        # unfolding the splits\n        batch, hidden = x.shape\n        x = x.reshape(batch // self.d_split, hidden * self.d_split)\n\n        # head processing\n        x = self.head(x)\n\n        return x\n    \n#---- CELL OUTPUT ------------:\nprint(\"Model class defined.\")","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:39.071439Z","iopub.execute_input":"2024-03-24T15:23:39.071891Z","iopub.status.idle":"2024-03-24T15:23:39.08836Z","shell.execute_reply.started":"2024-03-24T15:23:39.07186Z","shell.execute_reply":"2024-03-24T15:23:39.087613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Make sure model basically works\n\ndef test_GRU_works():\n    m = GRU().to(DEVICE)\n    x, y = next(iter(TRAIN_dl))\n    o = m(x.to(DEVICE))\n    print(f'Output of model...  {o.shape=}')\n    \ntest_GRU_works()\n\n#---- CELL OUTPUT ------------:\nprint(\"GRU model doesn't throw errors.\")","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:23:44.128911Z","iopub.execute_input":"2024-03-24T15:23:44.129303Z","iopub.status.idle":"2024-03-24T15:23:44.895345Z","shell.execute_reply.started":"2024-03-24T15:23:44.129274Z","shell.execute_reply":"2024-03-24T15:23:44.89346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create model\n\nGC()\n\nMODEL = GRU().to(DEVICE)\nOPT = t.optim.Adam(MODEL.parameters(), lr=1e-3, weight_decay=1e-5)\nprint(f'Model has {sum(p.numel() for p in MODEL.parameters())} params')","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:21:49.521787Z","iopub.status.idle":"2024-03-24T15:21:49.522191Z","shell.execute_reply.started":"2024-03-24T15:21:49.522001Z","shell.execute_reply":"2024-03-24T15:21:49.52202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAIN","metadata":{}},{"cell_type":"code","source":"def train(n_epochs, train_dataloader=TRAIN_dl, val_dataloader=VAL_dl, do_eval=True, device=DEVICE):\n    kl_loss = nn.KLDivLoss(reduction=\"batchmean\")\n    \n    for epoch in range(n_epochs):\n        \n        #TRAIN\n        train_loss = 0.0\n        batches_tq = tqdm(train_dataloader)\n        MODEL.train()\n        for x_train, y_train in batches_tq:\n            x_train, y_train = x_train.to(device), y_train.to(device)\n            logs = MODEL(x_train).log_softmax(-1)\n            loss = kl_loss(logs, y_train)\n            \n            OPT.zero_grad()\n            loss.backward()\n            OPT.step()\n\n            train_loss += loss.item()\n            batches_tq.set_description(f'Epoch {epoch+1}/{n_epochs}, Loss: {loss:.4f}')\n        train_loss /= len(train_dataloader)\n        \n        #CHECK VALIDATION PERFORMANCE\n        if do_eval:\n            MODEL.eval()            \n            val_loss = 0.0\n            with t.no_grad():  # No gradients needed for validation\n                for x_val, y_val in val_dataloader:\n                    x_val, y_val = x_val.to(device), y_val.to(device)\n                    logs = MODEL(x_val).log_softmax(-1)\n                    loss = kl_loss(logs, y_val)                \n                    val_loss += loss.item()    \n            MODEL.train() \n        val_loss /= len(val_dataloader)\n    \n        #UPDATE RESULTS\n        print(f'Epoch {epoch+1}/{N_EPOCHS}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}')\n\n        # SAVE MODEL\n        if epoch % 10 == 0:  # save every n epochs\n            now = datetime.now().strftime(\"%Y-%m-%d_%Hh%M\")\n            #t.save(model.state_dict(), f'weights/gru-4-splits_{now}.pt')\n            \n#---- CELL OUTPUT ------------:\nprint(\"Training function defined.\")            \n","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:21:49.52401Z","iopub.status.idle":"2024-03-24T15:21:49.524621Z","shell.execute_reply.started":"2024-03-24T15:21:49.524299Z","shell.execute_reply":"2024-03-24T15:21:49.524326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TRAIN!\n\nN_EPOCHS = 2\ntrain(N_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2024-03-24T15:21:49.526884Z","iopub.status.idle":"2024-03-24T15:21:49.527455Z","shell.execute_reply.started":"2024-03-24T15:21:49.52727Z","shell.execute_reply":"2024-03-24T15:21:49.52729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### WITH BATCH SIZE AT 32, takes about 12 mins/epoch:\n\n1.3778 at 39/3004\n1.3946 at 162/3004\n\nEpoch 2, Train Loss: 1.3810, Val Loss: 1.3768\nEpoch 3, Train Loss: 1.3810, Val Loss: 1.3764\n        \n**Maybe this was resetting model each time ran train? Changed train function to directly modify global MODEL and OPT\n\n### WITH BATCH SIZE AT 64, maybe 13 mins/epoch:\n\nEpoch 1/2, Train Loss: 1.3807, Val Loss: 1.3765\nEpoch 2/2, Train Loss: 1.3808, Val Loss: 1.3766\n\n### WITH BATCH SIZE AT 16, about 13 mins/epoch\n\nEpoch 1/2, Train Loss: 1.3810, Val Loss: 1.3763\nEpoch 2/2, Train Loss: 1.3807, Val Loss: 1.3764\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-03-24T03:45:06.596469Z","iopub.status.idle":"2024-03-24T03:45:06.596926Z","shell.execute_reply.started":"2024-03-24T03:45:06.596695Z","shell.execute_reply":"2024-03-24T03:45:06.596717Z"}}},{"cell_type":"markdown","source":"# THINGS TO TRY\n\n* Bigger batches to train faster?\n* Train longer\n* Are classes weighted right now?\n* Look at examples of what its getting right and wrong for patterns\n* Look at troubleshooting notes on remnote\n* clean eeg data in standard ways\n* incorporate spectrum data\n* ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ORIG NOTEBOOK:","metadata":{}},{"cell_type":"code","source":"def butter_bandpass(lowcut, highcut, fs, order=5):\n    return butter(order, [lowcut, highcut], fs=fs, btype='band')\n\ndef butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n    b, a = butter_bandpass(lowcut, highcut, fs, order=order)\n    y = lfilter(b, a, data)\n    return y\n\n\ndef denoise_filter(x):\n    # Sample rate and desired cutoff frequencies (in Hz).\n    fs = 200.0\n    lowcut = 1.0\n    highcut = 25.0\n    \n    # Filter a noisy signal.\n    T = 50\n    nsamples = T * fs\n    t = np.arange(0, nsamples) / fs\n    y = butter_bandpass_filter(x, lowcut, highcut, fs, order=6)\n    y = (y + np.roll(y,-1)+ np.roll(y,-2)+ np.roll(y,-3))/4\n    y = y[0:-1:4]\n    \n    return y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_TRAINING=False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_TRAINING:\n    eegs_data = np.load('/kaggle/input/hms-eeg-raw-dataset/eeg_specs.npy',allow_pickle=True).item()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataloader","metadata":{"papermill":{"duration":0.028035,"end_time":"2024-01-16T12:13:50.519598","exception":false,"start_time":"2024-01-16T12:13:50.491563","status":"completed"},"tags":[]}},{"cell_type":"code","source":"NAMES = ['LL','LP','RP','RR']\n\nFEATS = [['Fp1','F7','T3','T5','O1'],\n         ['Fp1','F3','C3','P3','O1'],\n         ['Fp2','F8','T4','T6','O2'],\n         ['Fp2','F4','C4','P4','O2']]\nPATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\n\nclass CustomDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        \n        row = self.dataframe.iloc[idx]\n        eeg_id = row['eeg_id']\n        eeg_sub_id = row['eeg_sub_id']\n        eeg_key = f'{eeg_id}_{eeg_sub_id}'\n        signals = eegs_data[eeg_key]\n        labels = row[TARGETS].values.astype(np.float64) #np.array(row[-6:]).reshape(6,1)\n        labels = labels/np.sum(labels)\n        return torch.tensor(signals,dtype=torch.float64), torch.tensor(labels,dtype=torch.float64)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the 1D CNN model\nclass CNN1D(nn.Module):\n    def __init__(self,in_channels):\n        super(CNN1D, self).__init__()\n        self.hidden_channels = 128\n        self.conv1 = nn.Conv1d(in_channels, 64, 20, 10)\n        self.conv2 = nn.Conv1d(64, 64, 10, 5)\n        self.conv3 = nn.Conv1d(64, 64, 12, 4)\n#         self.conv3 = nn.Conv1d(32, 32, 98, 1)\n        self.flatten = nn.Flatten()\n#         self.pool = nn.MaxPool1d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(640, 32)\n        self.fc2 = nn.Linear(32, 6)  # Adjust the input size based on your input dimensions\n        self.softmax = nn.Softmax(dim=1)\n        self.dropout = nn.Dropout(0.2)\n\n    def forward(self, x):\n#         print(x.shape)\n        x = F.relu(self.conv1(x))\n        x = self.dropout(x)\n#         print(x.shape)\n        x = F.relu(self.conv2(x))\n        x = self.dropout(x)\n#         print(x.shape)\n        x = F.relu(self.conv3(x))\n        x = self.dropout(x)\n#         print(x.shape)\n        x = (self.flatten(x))\n#         print(x.shape)\n#         x = x.view(-1, self.num_flat_features(x))\n#         print(x.shape)\n        x = self.fc1(x)\n        x = self.fc2(x)\n#         print(x)\n        x = self.softmax(x)\n#         print(x)\n        return x\n\n    def num_flat_features(self, x):\n        size = x.size()[1:]\n        num_features = 1\n        for s in size:\n            num_features *= s\n        return num_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an instance of your custom dataset\ndataset = CustomDataset(dataframe=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nexpert_consensus = [TARS[train.iloc[i,8]] for i in range(len(train))]\n\nclass_sample_count = np.array(\n    [len(np.where(expert_consensus == t)[0]) for t in np.unique(expert_consensus)])\n\nweight = 1. / class_sample_count\nsamples_weight = np.array([weight[t] for t in expert_consensus])\nsamples_weight = torch.from_numpy(samples_weight)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampler = torch.utils.data.sampler.WeightedRandomSampler(samples_weight.type('torch.DoubleTensor'), len(samples_weight))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('CNN1D_Model'):\n        os.makedirs('CNN1D_Model')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataLoader to handle batching and shuffling\nbatch_size = 256\ntrain_dataloader = DataLoader(dataset, batch_size=batch_size, sampler=sampler)\n\n# Create an instance of the model\nmodel = CNN1D(in_channels=4).double()\n# Define KL Divergence loss\ncriterion = nn.KLDivLoss(reduction=\"batchmean\")\n# Define optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nmodel.to(device)\nif IS_TRAINING:\n    model.train()    \n    epochs = 2\n    for epoch in range(epochs):\n        pbar = tqdm(train_dataloader)\n        for batch in pbar:\n            eeg_, label = batch\n            pred = model(eeg_.to(device))\n            loss = criterion(torch.log(pred), label.to(device))\n            # Backward pass and optimization\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n    #         stop\n\n        # Print loss for monitoring training progress\n            pbar.set_description('Batch loss{:.3f}'.format(loss.item()))\n#         print(f\"Epoch {epoch+1}/{epochs}, Loss: {loss.item()}\")\n\n    \n        torch.save(model.state_dict(), f'CNN1D_Model/model_{epoch}.pt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit to Kaggle LB\n","metadata":{"papermill":{"duration":0.039743,"end_time":"2024-01-16T12:14:15.75862","exception":false,"start_time":"2024-01-16T12:14:15.718877","status":"completed"},"tags":[]}},{"cell_type":"code","source":"del train; gc.collect()\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\nprint('Test shape:',test.shape)\ntest.head()","metadata":{"papermill":{"duration":0.059958,"end_time":"2024-01-16T12:14:15.85819","exception":false,"start_time":"2024-01-16T12:14:15.798232","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = '/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/'\nclass CustomDataset_test(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        \n        row = self.dataframe.iloc[idx]\n        eeg_id = row['eeg_id']\n        parq_path = f'{test_path}{eeg_id}.parquet'\n        eeg = pd.read_parquet(parq_path)\n        rows = len(eeg)\n        offset = (rows-10_000)//2\n        eeg = eeg.iloc[offset:offset+10_000]\n        \n        signals = []\n        for k in range(4):\n            COLS = FEATS[k]\n\n            # COMPUTE PAIR DIFFERENCES AND AVERAGE\n            x = eeg[COLS[0]].values - eeg[COLS[1]].values\n            for j in range(3):\n                x += eeg[COLS[j+1]].values - eeg[COLS[j+2]].values\n            x /= 4.0\n            x = denoise_filter(x)\n            signals.append(x)\n        signals = np.array(signals)\n        \n        return torch.tensor(signals,dtype=torch.float64)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_test = CustomDataset_test(dataframe=test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(dataset_test, batch_size=16,shuffle=False)\nmodel = CNN1D(in_channels=4).double()\nmodel.load_state_dict(torch.load('/kaggle/input/cnn-drop/model_1.pt'))\nmodel.eval()\nmodel.cpu()\npreds = []\n\nfor batch in test_loader:\n    pred = model(batch)\n    preds.append(pred.detach().numpy())\npreds = np.vstack(preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE SUBMISSION.CSV\nfrom IPython.display import display\n\nsub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = preds\nsub.to_csv('submission.csv',index=False)\nprint('Submission shape',sub.shape)\ndisplay( sub.head() )\n\n# SANITY CHECK TO CONFIRM PREDICTIONS SUM TO ONE\nprint('Sub row 0 sums to:',sub.iloc[0,-6:].sum())","metadata":{"papermill":{"duration":0.07372,"end_time":"2024-01-16T12:14:18.05987","exception":false,"start_time":"2024-01-16T12:14:17.98615","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}