{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":8102226,"sourceType":"datasetVersion","datasetId":4784951}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport io\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nimport random\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:00.851601Z","iopub.execute_input":"2024-04-23T15:35:00.851989Z","iopub.status.idle":"2024-04-23T15:35:06.500421Z","shell.execute_reply.started":"2024-04-23T15:35:00.851949Z","shell.execute_reply":"2024-04-23T15:35:06.499488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\n\ndef set_seed(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    torch.cuda.manual_seed(seed_value)\n    torch.backends.cudnn.deterministic = True \n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:06.502438Z","iopub.execute_input":"2024-04-23T15:35:06.502875Z","iopub.status.idle":"2024-04-23T15:35:06.511094Z","shell.execute_reply.started":"2024-04-23T15:35:06.502850Z","shell.execute_reply":"2024-04-23T15:35:06.510208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare df","metadata":{}},{"cell_type":"code","source":"# path for train and test\nbase_path = \"/kaggle/input/hms-harmful-brain-activity-classification\"\ntrain_csv_path = os.path.join(base_path, 'train.csv')\ntest_csv_path = os.path.join(base_path, 'test.csv')\n\n# train\ndf = pd.read_csv(train_csv_path)\n\n# trim data\ntemp = df[['eeg_id', 'expert_consensus']].drop_duplicates()\ndf = df.loc[temp.index].reset_index(drop=True)\n\n# add eeg & spec paths\ndf['eeg_path'] = base_path + '/train_eegs/' + df['eeg_id'].astype(str) + '.parquet'\ndf['spec_path'] = base_path + '/train_spectrograms/' + df['spectrogram_id'].astype(str) + '.parquet'\ndf['class_name'] = df['expert_consensus'].copy()\n\n# Define mappings\nclass_name_to_label = {\n    'Seizure': 0,\n    'GPD': 2,\n    'LRDA': 3,\n    'Other': 5,\n    'GRDA': 4,\n    'LPD': 1\n}\n\n# Apply mapping to create 'class_label'\ndf['class_label'] = df['class_name'].map(class_name_to_label)\n\n# test\ntest = pd.read_csv(test_csv_path)\ntest['eeg_path'] = base_path + '/test_eegs/' + test['eeg_id'].astype(str) + '.parquet'\ntest['spec_path'] = base_path + '/test_spectrograms/' + test['spectrogram_id'].astype(str) + '.parquet'\nif 'spectrogram_label_offset_seconds' not in test.columns:\n    test['spectrogram_label_offset_seconds'] = 0\nif 'class_label' not in test.columns:\n    test['class_label'] = 0\n\n# take a few samples as the test set\neeg_ids = []\nfor class_label in df['class_label'].unique():\n    eeg_id = df[df['class_label'] == class_label].iloc[0]['eeg_id']\n    eeg_ids.append(eeg_id)\n\ntest_df = df[df['eeg_id'].isin(eeg_ids)] # for test\ndf = df[~df['eeg_id'].isin(eeg_ids)] # train & val","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:06.515120Z","iopub.execute_input":"2024-04-23T15:35:06.515361Z","iopub.status.idle":"2024-04-23T15:35:06.886643Z","shell.execute_reply.started":"2024-04-23T15:35:06.515339Z","shell.execute_reply":"2024-04-23T15:35:06.885876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import model","metadata":{}},{"cell_type":"code","source":"# define efficientnet structure\nfrom torchvision import models, transforms\nimport torch.nn as nn\n\nmodel = models.efficientnet_v2_l(weights=None)\nnum_classes = 6\nnum_features = model.classifier[1].in_features\nmodel.classifier[1] = nn.Linear(num_features, num_classes)\n\n# Load pre-trained weights except for the classifier layer\nweights_path = '/kaggle/input/efficientnet-v2-l/efficientnet_v2_l-59c71312.pth'\nstate_dict = torch.load(weights_path, map_location='cpu')\nstate_dict = {k: v for k, v in state_dict.items() if not k.startswith('classifier.1')}\nmodel.load_state_dict(state_dict, strict=False)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:06.890022Z","iopub.execute_input":"2024-04-23T15:35:06.890426Z","iopub.status.idle":"2024-04-23T15:35:16.877238Z","shell.execute_reply.started":"2024-04-23T15:35:06.890392Z","shell.execute_reply":"2024-04-23T15:35:16.876434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare data for EfficientNet","metadata":{}},{"cell_type":"code","source":"# Get eeg_subset and spec_subset\ndef read_parquet_subset(parquet_file_path, offset_seconds, length, is_eeg=True):\n    offset_seconds = int(offset_seconds)  # Ensure offset_seconds integer, currently it's float\n    start_row = int(offset_seconds * 200) if is_eeg else int(offset_seconds / 2) # The starting row\n    end_row = start_row + (10000 if is_eeg else 300)  # The ending row\n    df = pd.read_parquet(parquet_file_path)\n    df_subset = df.iloc[start_row:end_row]  # Select the subset based on start and end row indices\n    return df_subset\n\n# Convert from 2d to 3d and replace null with 0\ndef convert_2d_to_3d(data_2d):\n    # Convert the 2D array into a 3D array with shape (height, width, channels)\n    data_2d = data_2d.to_numpy()\n    data_2d_clipped = np.clip(data_2d, 0, 255) # ensure value falls within 0 and 255\n    data_2d_clipped = np.nan_to_num(data_2d_clipped) # fillna with 0 (for numpy)\n    data_3d = np.repeat(data_2d_clipped[:, :, np.newaxis], 3, axis=2)\n    data_3d_uint8 = data_3d.astype(np.uint8)\n    return data_3d_uint8\n\n# generate feature vecotrs and labels\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\n\nclass SPECTROGRAM_Dataset(Dataset):\n    def __init__(self, dataframe, transform):\n        self.dataframe = dataframe\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        row = self.dataframe.iloc[idx]\n        spec_data = read_parquet_subset(row['spec_path'], row['spectrogram_label_offset_seconds'], 300, is_eeg=False)\n        spec_data.drop('time', axis=1, inplace=True)\n        spec_data_3d = convert_2d_to_3d(spec_data)\n        data_img = Image.fromarray(spec_data_3d)\n        feature_vector = self.transform(data_img)\n        label = row['class_label']\n        return feature_vector, label\n    \n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:16.878320Z","iopub.execute_input":"2024-04-23T15:35:16.878763Z","iopub.status.idle":"2024-04-23T15:35:16.891774Z","shell.execute_reply.started":"2024-04-23T15:35:16.878737Z","shell.execute_reply":"2024-04-23T15:35:16.891100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params\nlr=0.001\nnum_epochs = 100\nbatch_size=32\nnum_classes = 6\nnum_workers=2","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:16.892788Z","iopub.execute_input":"2024-04-23T15:35:16.893037Z","iopub.status.idle":"2024-04-23T15:35:16.906409Z","shell.execute_reply.started":"2024-04-23T15:35:16.893015Z","shell.execute_reply":"2024-04-23T15:35:16.905532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Val Split\n# use stratifiedgroupkfold to split train and val set to ensure balance in classes\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nlabels = df['class_label'].values\ngroups = df['eeg_id'].values\n\nsgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n\nfor train_idx, val_idx in sgkf.split(X=df, y=labels, groups=groups):\n    train_df, val_df = df.iloc[train_idx], df.iloc[val_idx]\n    \n# generate feature vectors and labels\ntrain_dataset = SPECTROGRAM_Dataset(train_df, transform)\nval_dataset = SPECTROGRAM_Dataset(val_df, transform)\ntest_dataset = SPECTROGRAM_Dataset(test_df, transform)\ntest2_dataset = SPECTROGRAM_Dataset(test, transform)\n\n# Initialize DataLoaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest2_loader = DataLoader(test2_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\nprint(f\"Size of train: {len(train_dataset)}\")\nprint(f\"Size of val: {len(val_dataset)}\")\nprint(f\"Size of test: {len(test_dataset)}\")\nprint(f\"Size of test2: {len(test2_dataset)}\")\nprint(f\"Number of train batches: {len(train_loader)}\")\nprint(f\"Number of val batches: {len(val_loader)}\")\nprint(f\"Number of test batches: {len(test_loader)}\")\nprint(f\"Number of test2 batches: {len(test2_loader)}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:16.907670Z","iopub.execute_input":"2024-04-23T15:35:16.908007Z","iopub.status.idle":"2024-04-23T15:35:24.802455Z","shell.execute_reply.started":"2024-04-23T15:35:16.907973Z","shell.execute_reply":"2024-04-23T15:35:24.801552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and Validation","metadata":{"execution":{"iopub.status.busy":"2024-04-08T04:48:06.892230Z","iopub.execute_input":"2024-04-08T04:48:06.893100Z","iopub.status.idle":"2024-04-08T04:48:06.897058Z","shell.execute_reply.started":"2024-04-08T04:48:06.893070Z","shell.execute_reply":"2024-04-08T04:48:06.895971Z"}}},{"cell_type":"code","source":"import torch.nn.functional as F\nfrom torch.optim import Adam\nimport time\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\nfrom torch.cuda.amp import autocast, GradScaler\n\n# convert integer class labels to one-hot vectors\ndef to_one_hot(labels, num_classes):\n    return torch.eye(num_classes, device=labels.device)[labels]\n\ncriterion = torch.nn.KLDivLoss(reduction='batchmean')\noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\nscheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=0, last_epoch=-1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:24.803762Z","iopub.execute_input":"2024-04-23T15:35:24.804138Z","iopub.status.idle":"2024-04-23T15:35:24.819051Z","shell.execute_reply.started":"2024-04-23T15:35:24.804105Z","shell.execute_reply":"2024-04-23T15:35:24.818084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_val_loss = float('inf')\ntraining_losses = []\nvalidation_losses = []\nepochs_no_improve = 0\nn_patience = 10\nepochs_no_improve = 0\naccumulation_steps = 4\nscaler = GradScaler()\n\nfor epoch in range(num_epochs):\n    model.train()\n    train_loss = 0.0\n    optimizer.zero_grad()\n    batch_count = 0\n    for i, (features, labels) in enumerate(train_loader):\n        features, labels = features.to(device), labels.to(device)\n            \n        with autocast():\n            outputs = model(features)\n            log_probs = F.log_softmax(outputs, dim=1)# Convert to log probabilities\n            one_hot_labels = to_one_hot(labels, num_classes=num_classes) # Convert labels to one-hot for KLDivLoss\n            loss = criterion(log_probs, one_hot_labels) / accumulation_steps # Normalize loss to account for accumulation\n        \n        scaler.scale(loss).backward()\n            \n        if (i + 1) % accumulation_steps == 0 or (i + 1) == len(train_loader):\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n                \n        train_loss += loss.item() * features.size(0)      \n        # Print every 100 batches to check progress\n        batch_count += 1\n        if batch_count % 100 == 0:\n            print(f'Epoch {epoch+1}, Batch {batch_count}, Loss: {loss.item():.4f}')  \n            \n    # Average training loss for the epoch        \n    train_loss /= len(train_loader.dataset)\n    training_losses.append(train_loss)\n    \n        \n    # Validation phase\n    model.eval()\n    val_loss = 0.0\n    with torch.no_grad(), autocast():\n        for features, labels in val_loader:\n            features = features.to(device)\n            labels = labels.to(device)\n            outputs = model(features)\n            log_probs = F.log_softmax(outputs, dim=1)\n            one_hot_labels = to_one_hot(labels, num_classes=num_classes)\n            loss = criterion(log_probs, one_hot_labels)\n            val_loss += loss.item() * features.size(0)\n    \n    # Average validation loss for the epoch\n    val_loss /= len(val_loader.dataset)\n    validation_losses.append(val_loss)\n    scheduler.step(val_loss)\n\n    print(f'Epoch [{epoch+1}/{num_epochs}], Training Loss: {train_loss:.4f}, Validation Loss: {val_loss:.4f}')\n    \n    # find smallest val loss to save as best model\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        epochs_no_improve = 0 # reset to 0\n        torch.save(model.state_dict(), 'best_model.pth')  # Save best model\n        print(f\"Best model saved with Validation Loss: {val_loss:.4f}\")\n    else:\n        epochs_no_improve += 1\n        \n    if epochs_no_improve == n_patience:\n        print(\"Early stopping triggered.\")\n        break","metadata":{"execution":{"iopub.status.busy":"2024-04-23T15:35:24.822352Z","iopub.execute_input":"2024-04-23T15:35:24.822692Z","iopub.status.idle":"2024-04-23T16:34:36.696993Z","shell.execute_reply.started":"2024-04-23T15:35:24.822665Z","shell.execute_reply":"2024-04-23T16:34:36.695821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Performance","metadata":{}},{"cell_type":"code","source":"# validate on test samples i extracted\n\nmodel.load_state_dict(torch.load('best_model.pth'))\nmodel.eval()\n\nall_preds = []\nall_targets = []\nall_probs = []\n\nwith torch.no_grad():\n    for features, labels in test_loader:\n        features = features.to(device).float()\n        labels = labels.to(device)\n        outputs = model(features)\n        \n        # store predicted labels, probabilites, and actual labels\n        probs = F.softmax(outputs, dim=1)\n        _, preds = torch.max(probs, 1)\n        all_preds.extend(preds.cpu().numpy())\n        all_probs.extend(probs.cpu().numpy())\n        all_targets.extend(labels.cpu().numpy())\n\n# classification report        \nfrom sklearn.metrics import classification_report\nclassification_report(all_targets, all_preds, target_names=[\"Seizure\", \"LPD\", \"GPD\", \"LRDA\", \"GRDA\", \"Other\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:36.699237Z","iopub.execute_input":"2024-04-23T16:34:36.700162Z","iopub.status.idle":"2024-04-23T16:34:38.090854Z","shell.execute_reply.started":"2024-04-23T16:34:36.700123Z","shell.execute_reply":"2024-04-23T16:34:38.089761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(all_preds)\nprint(all_targets)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:38.092330Z","iopub.execute_input":"2024-04-23T16:34:38.092617Z","iopub.status.idle":"2024-04-23T16:34:38.097642Z","shell.execute_reply.started":"2024-04-23T16:34:38.092591Z","shell.execute_reply":"2024-04-23T16:34:38.096661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"# get pred prob of test\n\nmodel.load_state_dict(torch.load('best_model.pth'))\nmodel.eval()\n\nall_preds = []\nall_probs = []\n\nwith torch.no_grad():\n    for features, labels in test2_loader:\n        features = features.to(device).float()\n        labels = labels.to(device)\n        outputs = model(features)\n        \n        # store predicted labels, probabilites\n        probs = F.softmax(outputs, dim=1)\n        _, preds = torch.max(probs, 1)\n        all_preds.extend(preds.cpu().numpy())\n        all_probs.extend(probs.cpu().numpy())\n\nprint(all_preds)\nprint(all_probs)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:38.098797Z","iopub.execute_input":"2024-04-23T16:34:38.099079Z","iopub.status.idle":"2024-04-23T16:34:39.067924Z","shell.execute_reply.started":"2024-04-23T16:34:38.099044Z","shell.execute_reply":"2024-04-23T16:34:39.066770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample submission\nsample_submission_csv_path = os.path.join(base_path, 'sample_submission.csv')\nsub = pd.read_csv(sample_submission_csv_path)\nsub","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:39.072906Z","iopub.execute_input":"2024-04-23T16:34:39.073311Z","iopub.status.idle":"2024-04-23T16:34:39.103337Z","shell.execute_reply.started":"2024-04-23T16:34:39.073282Z","shell.execute_reply":"2024-04-23T16:34:39.102447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(all_probs, columns=['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'])\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:39.104452Z","iopub.execute_input":"2024-04-23T16:34:39.104779Z","iopub.status.idle":"2024-04-23T16:34:39.117720Z","shell.execute_reply.started":"2024-04-23T16:34:39.104747Z","shell.execute_reply":"2024-04-23T16:34:39.116427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['eeg_id'] = test['eeg_id']  \nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:39.118930Z","iopub.execute_input":"2024-04-23T16:34:39.119288Z","iopub.status.idle":"2024-04-23T16:34:39.130272Z","shell.execute_reply.started":"2024-04-23T16:34:39.119265Z","shell.execute_reply":"2024-04-23T16:34:39.129462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.iloc[:,:6].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:39.131556Z","iopub.execute_input":"2024-04-23T16:34:39.131905Z","iopub.status.idle":"2024-04-23T16:34:39.142939Z","shell.execute_reply.started":"2024-04-23T16:34:39.131873Z","shell.execute_reply":"2024-04-23T16:34:39.142032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T16:34:39.143973Z","iopub.execute_input":"2024-04-23T16:34:39.144260Z","iopub.status.idle":"2024-04-23T16:34:39.152183Z","shell.execute_reply.started":"2024-04-23T16:34:39.144221Z","shell.execute_reply":"2024-04-23T16:34:39.151355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}