{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":11695524,"sourceType":"datasetVersion","datasetId":7340660},{"sourceId":11700199,"sourceType":"datasetVersion","datasetId":7343910},{"sourceId":11728815,"sourceType":"datasetVersion","datasetId":7362525},{"sourceId":11728840,"sourceType":"datasetVersion","datasetId":7362544},{"sourceId":176545100,"sourceType":"kernelVersion"},{"sourceId":377198,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":311515,"modelId":331893}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport copy\nfrom tqdm.auto import tqdm\nimport math\nimport random\n\n# EEG Processing\nimport mne\nfrom scipy.signal import butter, lfilter\n\n# Torch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchaudio\nfrom torchinfo import summary\nimport torchvision.transforms as T # Added for resizing\nfrom torch.cuda.amp import GradScaler, autocast\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import GroupKFold, StratifiedGroupKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom collections import Counter\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-08T07:57:42.618740Z","iopub.execute_input":"2025-05-08T07:57:42.619123Z","iopub.status.idle":"2025-05-08T07:57:59.585885Z","shell.execute_reply.started":"2025-05-08T07:57:42.619086Z","shell.execute_reply":"2025-05-08T07:57:59.584509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cp /kaggle/input/2nd-place-solution/x3d.py .\nfrom x3d import create_x3d","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-08T07:59:54.081391Z","iopub.execute_input":"2025-05-08T07:59:54.081793Z","iopub.status.idle":"2025-05-08T07:59:54.266935Z","shell.execute_reply.started":"2025-05-08T07:59:54.081765Z","shell.execute_reply":"2025-05-08T07:59:54.265793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-08T07:59:56.473821Z","iopub.execute_input":"2025-05-08T07:59:56.474734Z","iopub.status.idle":"2025-05-08T07:59:56.485613Z","shell.execute_reply.started":"2025-05-08T07:59:56.474687Z","shell.execute_reply":"2025-05-08T07:59:56.484144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Configuration\nCFG = {\n    'data_dir': '/kaggle/input/hms-harmful-brain-activity-classification',\n    'train_csv': '/kaggle/input/hms-harmful-brain-activity-classification/train.csv',\n    'train_eeg_dir': '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs',\n    'processed_eeg_dir_50': '/kaggle/input/eeg-spec-61-75-50/processed_eeg_spectrograms_61_75_middle_50',\n    'processed_eeg_dir_10': '/kaggle/input/eeg-spec-61-75-10/processed_eeg_spectrograms_61_75_middle_10',\n    'model_path': '/kaggle/input/eeg_3d_cnn_torch/pytorch/default/1/x3d_model_fold0_best.pth',\n    'use_saved_model': False,\n    # Add other paths if needed (like for spectrograms if you extend the model)\n\n    'seed': 42,\n    'num_folds': 5,\n    'selected_fold': 0, # Train only one fold in this example\n    'epochs': 10, # Adjust number of epochs\n    'batch_size': 16, # Adjust based on GPU memory\n    'num_workers': 2,\n    'lr': 5e-5,\n    'weight_decay': 8e-4,\n    'patience': 3, # For early stopping (optional)\n\n    # Model Params\n    'model_name': 'x3d_m',\n    'x3d_input_clip_length': 16, # Matches the 16 differential EEG channels after montage\n    'x3d_input_crop_size': 224, # 312 for x3d-l. Target H, W for the X3D model input\n    'x3d_depth_factor': 5.0, #5.0 for x3d-l\n    'num_classes': 6,\n    'target_cols': ['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote'],\n    'class_names': ['Other', 'Seizure', 'LPD', 'GPD', 'GRDA', 'LRDA'],\n    'use_amp': True\n}\n\n# Set Seed for Reproducibility\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(CFG['seed'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-08T07:59:58.526037Z","iopub.execute_input":"2025-05-08T07:59:58.526410Z","iopub.status.idle":"2025-05-08T07:59:58.543366Z","shell.execute_reply.started":"2025-05-08T07:59:58.526385Z","shell.execute_reply":"2025-05-08T07:59:58.542353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(CFG['train_csv'])\nTARGETS = df.columns[-6:]\n\ntrain = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_id':'first','spectrogram_label_offset_seconds':'min'})\ntrain.columns = ['spec_id','min']\n\ntmp = df.groupby('eeg_id')[['spectrogram_id','spectrogram_label_offset_seconds']].agg(\n    {'spectrogram_label_offset_seconds':'max'})\ntrain['max'] = tmp\n\ntmp = df.groupby('eeg_id')[['patient_id']].agg('first')\ntrain['patient_id'] = tmp\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n    train[t] = tmp[t].values\n    \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nsum_targets = tmp.sum(axis=1)\nmax_vote_percentage = tmp.max(axis=1) / sum_targets\ntrain['max_vote_percentage'] = max_vote_percentage\n\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\ntrain = train[train['max_vote_percentage']>=.75]\ntrain.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-08T08:00:00.627934Z","iopub.execute_input":"2025-05-08T08:00:00.628312Z","iopub.status.idle":"2025-05-08T08:00:01.124547Z","shell.execute_reply.started":"2025-05-08T08:00:00.628289Z","shell.execute_reply":"2025-05-08T08:00:01.123421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"frequency = train['target'].value_counts()\nprint(frequency)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AlaskaDataIter(Dataset):\n    def __init__(self, df, cfg, training_flag=True, flip=False):\n        self.df = df\n        self.cfg = cfg\n        self.training_flag = training_flag\n        self.transform = T.Compose([\n            T.RandomHorizontalFlip(),\n            T.RandomRotation(10)\n        ])\n        self.class_to_idx = {class_name: idx for idx, class_name in enumerate(self.cfg['class_names'])}\n\n    def __len__(self):\n        return len(self.df)   \n\n    def __getitem__(self, item):\n        dp = self.df.iloc[item]\n        \n        eeg_id = dp['eeg_id']\n        \n        # Load both spectrograms\n        spec_50_path = os.path.join(self.cfg['processed_eeg_dir_50'], f\"{eeg_id}.pt\")\n        # spec_10_path = os.path.join(self.cfg['processed_eeg_dir_10'], f\"{eeg_id}.pt\")\n        \n        spec_50 = torch.load(spec_50_path).float() #[16, 224, 224]\n        # spec_10 = torch.load(spec_10_path).float() #[16, 224, 224]\n        \n        if self.training_flag and random.random() > 0.5:\n            spec_50 = self.transform(spec_50)\n            # spec_10 = self.transform(spec_10)           \n\n        targets = dp[TARGETS].values.astype(np.float32)\n        # targets /= targets.sum() # Normalize\n        targets_tensor = torch.from_numpy(targets)\n\n        true_class_name = dp['target'] # This is 'expert_consensus' string\n        true_class_idx = self.class_to_idx[true_class_name]\n        true_class_idx_tensor = torch.tensor(true_class_idx, dtype=torch.long)\n\n        return spec_50, targets_tensor, true_class_idx_tensor\n\n# === Model Definition ===\nclass Modelx3d(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        try:\n             # Create X3D, potentially loading pretrained weights if available/desired\n             # NOTE: Check `create_x3d` documentation for pretrained weight options\n            self.net = create_x3d(\n                input_clip_length=cfg['x3d_input_clip_length'], # Corresponds to EEG channels (T)\n                input_crop_size=cfg['x3d_input_crop_size'],   # Corresponds to Freq (H) and Time (W) after resize\n                depth_factor=cfg['x3d_depth_factor']\n            )\n            # Modify final layers to use as feature extractor\n            # Check the actual structure of your X3D model using print(self.net)\n            # The layer names ('blocks', 'proj', etc.) might differ slightly\n            self.net.blocks[5].dropout = nn.Identity()\n            self.net.blocks[5].proj = nn.Identity()\n            self.net.blocks[5].activation = nn.Identity()\n            self.net.blocks[5].output_pool = nn.Identity() # Adjust if pooling name is different\n            self.feature_dim = 2048 # Output dim of X3D_L feature extractor (verify this)\n\n        except Exception as e:\n            print(f\"Error creating X3D model: {e}\")\n            print(\"Ensure 'create_x3d' parameters match the implementation in x3d.py\")\n            raise e\n\n    def forward(self, x):\n        x = self.net(x)\n        return x\n\nclass Netx3dTrain(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg        \n\n        # X3D Feature Extractor Backbone\n        self.model_backbone = Modelx3d(cfg)\n        \n        # self.fc = nn.Linear(self.model_backbone.feature_dim, cfg['num_classes'])\n        self.fc = nn.Sequential(nn.Dropout(0.3),\n                                nn.Linear(self.model_backbone.feature_dim, cfg['num_classes'], bias=True)\n                               )\n\n    def forward(self, spec_50):\n        bs = spec_50.size(0)\n        \n        # x = torch.cat([spec_10, spec_50], dim=1)\n        \n        x = torch.unsqueeze(spec_50, dim=1)       \n        \n        x = torch.cat([x, x, x], dim=1)        \n\n        features = self.model_backbone(x)\n        \n        features = features.view(bs, -1)\n\n        output = self.fc(features)\n\n        return output\n\n# === Training Utilities ===\n\ndef train_epoch(model, loader, optimizer, criterion, device, scaler=None, use_amp=False):\n    model.train()\n    train_loss = 0.0\n    all_preds = []\n    all_targets = []\n\n    pbar = tqdm(loader, desc=\"Training\", leave=False)\n    for batch_idx, (spec_50, targets, _) in enumerate(pbar):\n        # spec_10 = spec_10.to(device)\n        spec_50 = spec_50.to(device)\n        targets = targets.to(device) # Target shape [B, 6]\n\n        optimizer.zero_grad()\n\n        with autocast(enabled=use_amp):\n            logits = model(spec_50)\n            log_probs = torch.log_softmax(logits, dim=-1)\n            loss = criterion(log_probs, targets) # Assumes KLDivLoss\n\n        # <<< Scale loss and backpropagate using scaler >>>\n        if use_amp:\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            loss.backward()\n            optimizer.step()        \n\n        train_loss += loss.item()\n        all_preds.append(torch.softmax(logits, dim=-1).detach().cpu().numpy()) # Store probabilities\n        all_targets.append(targets.cpu().numpy())\n\n        pbar.set_postfix(loss=loss.item())\n        if batch_idx % 10 == 0:\n            print(f\"Batch [{batch_idx}/{len(loader)}], Loss: {loss.item():.4f}\")\n\n    avg_loss = train_loss / len(loader)\n    predictions = np.concatenate(all_preds)\n    true_labels = np.concatenate(all_targets)\n\n    # Calculate Accuracy (using argmax for simplicity, KLDiv is the competition metric)\n    acc = accuracy_score(np.argmax(true_labels, axis=1), np.argmax(predictions, axis=1))\n\n    return avg_loss, acc\n\ndef validate_epoch(model, loader, criterion, device, use_amp=False):\n    model.eval()\n    val_loss = 0.0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        pbar = tqdm(loader, desc=\"Validation\", leave=False)\n        for spec_50, targets, _ in pbar:\n            # spec_10 = spec_10.to(device)\n            spec_50 = spec_50.to(device)\n            targets = targets.to(device)\n\n            with autocast(enabled=use_amp):\n                logits = model(spec_50)\n                log_probs = torch.log_softmax(logits, dim=-1)\n                loss = criterion(log_probs, targets)\n\n            val_loss += loss.item()\n            all_preds.append(torch.softmax(logits, dim=-1).cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n            pbar.set_postfix(loss=loss.item())\n\n    avg_loss = val_loss / len(loader)\n    predictions = np.concatenate(all_preds)\n    true_labels = np.concatenate(all_targets)\n\n    # Calculate Accuracy\n    acc = accuracy_score(np.argmax(true_labels, axis=1), np.argmax(predictions, axis=1))\n\n    return avg_loss, acc\n\ndef run_training(cfg, train_df, val_df):\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Using device: {device}\")\n\n    # Create datasets\n    train_dataset = AlaskaDataIter(train_df, cfg, training_flag=True)\n    val_dataset = AlaskaDataIter(val_df, cfg, training_flag=False) # No augmentation for validation\n\n    print(\"\\nActual class distribution in validation dataset:\")\n    \n    all_true_class_indices = []\n    for i in range(len(val_dataset)):\n        _, _, true_class_idx = val_dataset[i]\n        all_true_class_indices.append(true_class_idx.item())\n    \n    target_counts = Counter(all_true_class_indices)\n    for class_idx_enum, class_name_enum in enumerate(cfg['class_names']):\n        print(f\"{class_name_enum}: {target_counts.get(class_idx_enum, 0)}\")\n    \n    # Create dataloaders\n    train_loader = DataLoader(train_dataset, batch_size=cfg['batch_size'], shuffle=True, \n                              num_workers=cfg['num_workers'], pin_memory=True, drop_last=True,\n                              persistent_workers=True, prefetch_factor=2 )\n    val_loader = DataLoader(val_dataset, batch_size=cfg['batch_size'] * 2, shuffle=False, \n                            num_workers=cfg['num_workers'], pin_memory=True, \n                            persistent_workers=True, prefetch_factor=2 )\n\n    # Initialize model\n    model = Netx3dTrain(cfg).to(device)\n    \n    # if cfg['use_saved_model']:\n    #     model.load_state_dict(torch.load(cfg['model_path'], map_location=device))\n    # else:\n    optimizer = optim.AdamW(model.parameters(), lr=cfg['lr'], weight_decay=cfg['weight_decay'])\n    \n    criterion = nn.KLDivLoss(reduction='batchmean')\n\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=cfg['epochs'], eta_min=1e-6)\n    \n    scaler = GradScaler(enabled=cfg['use_amp'])\n    \n    best_val_loss = float('inf')\n    best_epoch = -1\n    epochs_no_improve = 0\n\n    for epoch in range(cfg['epochs']):\n        print(f\"\\nEpoch {epoch+1}/{cfg['epochs']}\")\n\n        train_loss, train_acc = train_epoch(model, train_loader, optimizer, criterion, device, scaler=scaler, use_amp=cfg['use_amp'])\n        val_loss, val_acc = validate_epoch(model, val_loader, criterion, device, use_amp=cfg['use_amp']) # Add other metrics if calculated\n\n        if scheduler:\n            scheduler.step()\n\n        print(f\"Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}\")\n        print(f\"Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.4f}\") # Print other metrics\n\n        # Save best model checkpoint\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            best_epoch = epoch\n            epochs_no_improve = 0\n            model_path = f\"x3d_model_fold{cfg['selected_fold']}_best.pth\"\n            torch.save(model.state_dict(), model_path)\n            cfg['model_path'] = model_path\n            print(f\"Validation loss improved. Saved model to {model_path}\")\n        else:\n            epochs_no_improve += 1\n            print(f\"Validation loss did not improve for {epochs_no_improve} epoch(s).\")\n\n        # Early stopping (optional)\n        if cfg['patience'] > 0 and epochs_no_improve >= cfg['patience']:\n            print(f\"Early stopping triggered after {epoch+1} epochs.\")\n            break\n\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    print(f\"\\nTraining finished. Best validation loss {best_val_loss:.4f} at epoch {best_epoch+1}\")\n    # Load best model for potential further use/inference\n    # model.load_state_dict(torch.load(f\"x3d_model_fold{cfg['selected_fold']}_best.pth\"))\n    \n    \n    # model.load_state_dict(torch.load(cfg['model_path'], map_location=device))\n    model.eval()\n    \n    # Prepare the validation dataset and dataloader\n    val_dataset = AlaskaDataIter(val_df, cfg, training_flag=False)\n    val_loader = DataLoader(val_dataset, batch_size=cfg['batch_size'] * 2, shuffle=False, \n                            num_workers=cfg['num_workers'], pin_memory=True)\n\n    # Collect predictions and true labels\n    all_preds = []\n    all_labels = []\n\n    with torch.no_grad():\n        for spec_50, _, true_class_indices in val_loader:\n            # spec_10 = spec_10.to(device)\n            spec_50 = spec_50.to(device)\n            true_class_indices = true_class_indices.to(device)\n\n            logits = model(spec_50)\n            preds = torch.argmax(logits, dim=1)  # Get predicted class indices\n\n            all_preds.extend(preds.cpu().numpy())\n            all_labels.extend(true_class_indices.cpu().numpy())  # Convert one-hot to class indices\n\n    # classification report\n    print(classification_report(all_labels, all_preds, target_names=cfg['class_names'], zero_division=0))\n\n    # confusion matrix\n    cm = confusion_matrix(all_labels, all_preds)\n\n    # Plot confusion matrix\n    plt.figure(figsize=(10, 7))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n                xticklabels=cfg['class_names'], \n                yticklabels=cfg['class_names'])\n    plt.ylabel('Actual')\n    plt.xlabel('Predicted')\n    plt.title('Confusion Matrix')\n    plt.show()\n\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-08T03:11:46.640642Z","iopub.execute_input":"2025-05-08T03:11:46.640819Z","iopub.status.idle":"2025-05-08T03:13:39.376479Z","shell.execute_reply.started":"2025-05-08T03:11:46.640805Z","shell.execute_reply":"2025-05-08T03:13:39.375054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == '__main__':\n    print(\"Starting Training Script\")\n    print(\"Loading train.csv...\")\n    \n    # --- Data Splitting ---\n    print(f\"Setting up Fold {CFG['selected_fold']}...\")\n    gkf = GroupKFold(n_splits=CFG['num_folds'])\n    splits = list(gkf.split(X=train, y=train[TARGETS], groups=train['patient_id'])) # Split based on first target, grouped by patient\n\n    train_idx, val_idx = splits[CFG['selected_fold']]\n    train_fold_df = train.iloc[train_idx].reset_index(drop=True)\n    val_fold_df = train.iloc[val_idx].reset_index(drop=True)\n\n    print(f\"Train fold {CFG['selected_fold']} size: {len(train_fold_df)}\")\n    print(f\"Validation fold {CFG['selected_fold']} size: {len(val_fold_df)}\")\n\n    # --- Run Training ---\n    print(\"Starting training run...\")\n    trained_model = run_training(CFG, train_fold_df, val_fold_df)\n    print(\"Training run complete.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# logits = model(eeg) # Logits shape [B, 6]\n\n        # CrossEntropyLoss expects logits [B, C] and target indices [B]\n        # Our targets are probabilities [B, C]. Use KLDivLoss or convert targets.\n        # Option 1: KLDivLoss (requires log_softmax output from model)\n        # loss = criterion(torch.log_softmax(logits, dim=-1), targets)\n\n        # Option 2: Convert targets to class indices for CrossEntropyLoss\n        # This only works if one class is clearly dominant (not ideal for vote distributions)\n        # target_indices = torch.argmax(targets, dim=1)\n        # loss = criterion(logits, target_indices)\n\n        # Option 3: Use CrossEntropyLoss with soft labels (probabilities)\n        # This is equivalent to KLDivLoss between softmax(logits) and targets\n        # log_probs = torch.log_softmax(logits, dim=-1)\n        # loss = criterion(log_probs, targets) # criterion should be nn.KLDivLoss(reduction='batchmean')","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-05-08T03:13:39.377648Z","iopub.status.idle":"2025-05-08T03:13:39.378129Z","shell.execute_reply.started":"2025-05-08T03:13:39.377872Z","shell.execute_reply":"2025-05-08T03:13:39.377896Z"}},"outputs":[],"execution_count":null}]}