{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":106680,"databundleVersionId":13374319,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport numpy as np\nimport seaborn as sns\nimport pandas as pd\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom collections import Counter\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nfrom typing import List, Optional\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.decomposition import PCA # Import PCA\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import roc_auc_score\nfrom xgboost import plot_importance\n\nimport optuna","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:06.050843Z","iopub.execute_input":"2025-12-14T11:39:06.051329Z","iopub.status.idle":"2025-12-14T11:39:11.975470Z","shell.execute_reply.started":"2025-12-14T11:39:06.051301Z","shell.execute_reply":"2025-12-14T11:39:11.974755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **AIRR-ML🧬25 - EDA & Visualization**","metadata":{}},{"cell_type":"code","source":"PATH_DATASET = \"/kaggle/input/adaptive-immune-profiling-challenge-2025\"\nPATH_TRAIN_DATASETS = os.path.join(PATH_DATASET, 'train_datasets', 'train_datasets')\ntrain_datasets = sorted(os.listdir(PATH_TRAIN_DATASETS))\nprint(train_datasets)\nPATH_TEST_DATASETS = os.path.join(PATH_DATASET, 'test_datasets', 'test_datasets')\ntest_datasets = sorted(os.listdir(PATH_TEST_DATASETS))\nprint(test_datasets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:11.976800Z","iopub.execute_input":"2025-12-14T11:39:11.977335Z","iopub.status.idle":"2025-12-14T11:39:12.003923Z","shell.execute_reply.started":"2025-12-14T11:39:11.977304Z","shell.execute_reply":"2025-12-14T11:39:12.003269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RANDOM_SEED = 42\nN_SPLITS = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:12.004462Z","iopub.execute_input":"2025-12-14T11:39:12.004687Z","iopub.status.idle":"2025-12-14T11:39:12.013111Z","shell.execute_reply.started":"2025-12-14T11:39:12.004667Z","shell.execute_reply":"2025-12-14T11:39:12.012580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def parse_tsv_files(folder_path: str, feature_colums = ('v_call', 'j_call')):\n    folder = os.path.basename(folder_path) # Derive folder name from folder_path\n    # List all files in the directory\n    files = os.listdir(folder_path)\n\n    # Filter for .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    # Iterate through each TSV file, load it into a DataFrame, and print column names\n    dataset = []\n    for tsv_file in tqdm(tsv_files, desc=\"Loading TSV files\"):\n        file_path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n        try:\n            df = pd.read_csv(file_path, sep='\\t')\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n        \n        one_case = {\"ID\": file_name, \"dataset\": folder}\n        if metadata is not None:\n            one_case = {\"label_positive\": int(metadata.at[tsv_file, \"label_positive\"])}\n        for col in feature_colums:\n            counts = df[col].value_counts() / len(df)\n            one_case.update(counts.to_dict())\n            # print(one_case)\n            dataset.append(one_case)\n\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:12.013952Z","iopub.execute_input":"2025-12-14T11:39:12.014180Z","iopub.status.idle":"2025-12-14T11:39:12.034872Z","shell.execute_reply.started":"2025-12-14T11:39:12.014165Z","shell.execute_reply":"2025-12-14T11:39:12.034225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_tsv_files_export_parquet(folder_path: str, output_path: str, show_hist: Optional[List[str]] = None):\n    folder = os.path.basename(folder_path) # Derive folder name from folder_path\n    # List all files in the directory\n    files = os.listdir(folder_path)\n\n    # Filter for .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    # Iterate through each TSV file, load it into a DataFrame, and print column names\n    dfs = []\n    for tsv_file in tqdm(tsv_files, desc=\"Loading TSV files\"):\n        file_path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n        try:\n            df = pd.read_csv(file_path, sep='\\t')\n            df['repertoire_id'] = file_name\n            dfs.append(df)\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n\n    merged_df = pd.concat(dfs, ignore_index=True)\n    del dfs # Free up memory\n\n    print(f\"Merged DataFrame shape: {merged_df.shape}\")\n    for col in merged_df.columns:\n        print(f\"Unique values in column '{col}': {len(merged_df[col].unique())}\")\n    print(\"Merged DataFrame head:\")\n    display(merged_df.head())\n\n    os.makedirs(output_path, exist_ok=True)\n    merged_df.to_parquet(f'{output_path}/{folder}.parquet')\n\n    # Plot histograms for specified columns if show_hist is provided\n    if not isinstance(show_hist, list) and not show_hist:\n        return\n    print(f\"Plotting histograms for columns: {', '.join(show_hist)}\")\n    for col in show_hist:\n        if col not in merged_df.columns:\n            print(f\"Warning: Column '{col}' not found in the DataFrame for {folder}.\")\n            continue\n        # Get all value counts\n        all_counts = merged_df[col].value_counts()\n        \n        plt.figure(figsize=(min(12, len(all_counts) * 0.3), 4)) # Adjust figure size dynamically\n        sns.barplot(x=all_counts.index, y=all_counts.values, palette='viridis')\n        plt.title(f'Value Counts for {col} in {folder}')\n        plt.xlabel(col)\n        plt.ylabel('Count')\n        plt.xticks(rotation=90, ha='center') # Rotate labels more for many categories\n        plt.grid(True)\n        plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:12.036690Z","iopub.execute_input":"2025-12-14T11:39:12.036861Z","iopub.status.idle":"2025-12-14T11:39:12.062101Z","shell.execute_reply.started":"2025-12-14T11:39:12.036847Z","shell.execute_reply":"2025-12-14T11:39:12.061183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Training Dataset (EDA & Visualization)**","metadata":{}},{"cell_type":"code","source":"# Iterate over all sub-datasets\nfor folder in tqdm(train_datasets):\n    path_dataset_ = os.path.join(PATH_TRAIN_DATASETS, folder)\n    load_tsv_files_export_parquet(\n        path_dataset_, output_path='train_dataset', show_hist=['v_call', 'j_call', 'd_call'])\n    new_meta_csv = os.path.join(\"train_dataset\", f\"{folder}-metadata.csv\")\n    shutil.copy(os.path.join(path_dataset_, \"metadata.csv\"), new_meta_csv)\n    df_meta = pd.read_csv(new_meta_csv)\n    display(df_meta)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:39:12.062782Z","iopub.execute_input":"2025-12-14T11:39:12.063039Z","iopub.status.idle":"2025-12-14T11:50:04.973837Z","shell.execute_reply.started":"2025-12-14T11:39:12.063013Z","shell.execute_reply":"2025-12-14T11:50:04.972889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_kmers(seq: str, k: int = 5):\n    \"\"\"Return all k-mers from an amino-acid sequence.\"\"\"\n    if not isinstance(seq, str):\n        return []\n    seq = seq.strip()\n    if len(seq) < k:\n        return []\n    return [seq[i:i+k] for i in range(len(seq) - k + 1)]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def parse_tsv_files(\n        folder_path: str, \n        feature_columns=('v_call', 'j_call'),\n        kmer_column='junction_aa',\n        k=3  # length of kmers\n):\n    folder = os.path.basename(folder_path)\n    files = os.listdir(folder_path)\n\n    # Only .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    # Metadata\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    dataset = []\n\n    for tsv_file in tqdm(tsv_files):\n        path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n\n        try:\n            df = pd.read_csv(path, sep=\"\\t\")\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n            continue\n\n        # Collect all features here\n        one_case = {\"ID\": file_name, \"dataset\": folder}\n\n        if metadata is not None:\n            one_case[\"label_positive\"] = int(metadata.at[tsv_file, \"label_positive\"])\n\n        # === 1) Count V / J genes ===\n        for col in feature_columns:\n            counts = df[col].value_counts(normalize=True)  # normalized frequency %\n            one_case.update({f\"{col}__{k}\": v for k, v in counts.to_dict().items()})\n\n        # === 2) Count k-mers from junction_aa ===\n        if kmer_column in df.columns:\n            all_kmers = []\n\n            for seq in df[kmer_column].dropna():\n                all_kmers.extend(extract_kmers(seq, k=k))\n\n            kmer_counts = Counter(all_kmers)\n\n            # convert to relative frequencies\n            total = sum(kmer_counts.values())\n            if total > 0:\n                kmer_freqs = {f\"kmer{k}_{mer}\": count / total for mer, count in kmer_counts.items()}\n            else:\n                kmer_freqs = {}\n\n            one_case.update(kmer_freqs)\n\n        dataset.append(one_case)\n\n    return dataset","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(train_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TRAIN_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_train = pd.DataFrame(dataset)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_train.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f_cols = [col for col in dataset_train.columns if col not in ['ID', 'dataset']]\n\ncols_na = dataset_train[f_cols].isnull().sum()\n\ncols_na = cols_na[cols_na > 3100]\nexcluded_features = cols_na.index.to_list()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(excluded_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Testing Dataset (EDA & Visualization)**","metadata":{}},{"cell_type":"code","source":"# Iterate over all sub-datasets\nfor folder in tqdm(test_datasets):\n    path_dataset_ = os.path.join(PATH_TEST_DATASETS, folder)\n    load_tsv_files_export_parquet(\n        path_dataset_, output_path='test_dataset', show_hist=['v_call', 'j_call', 'd_call'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T11:50:04.974732Z","iopub.execute_input":"2025-12-14T11:50:04.974986Z","execution_failed":"2025-12-14T12:12:43.451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(test_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TEST_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_test = pd.DataFrame(dataset)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f_cols = [col for col in dataset_test.columns if col not in ['ID', 'dataset']]\n\n# 2. Sum of missing values\ncols_na = dataset_test[f_cols].isnull().sum()\n\n# 3. Just an attempt to reduce variables with many missing values\ncols_na = cols_na[cols_na > 4000]\nexcluded_features_test = cols_na.index.to_list()\n\nexcluded_features =  excluded_features  +  excluded_features_test\nexcluded_features = list(set(excluded_features))\nlen(excluded_features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"extra_excluded = [\"TCRBV22.X\",\"TCRBV29.X\",\"TCRBV25.or09_02\", \"TCRBV29.or09_02\", \n                      \"X\",\"TCRBV24\"]\nexcluded_features +=  extra_excluded\nexcluded_features = list(set(excluded_features))\nprint(f\"n_excluded: {len(excluded_features)}\")\nexcluded_features[:10]","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **XGBOOST TRAINING**","metadata":{}},{"cell_type":"code","source":"# Prepare the data\nX_train0 = dataset_train.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_train = dataset_train['label_positive']\n\nX_test = dataset_test.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_test = pd.Series([]) # Initialize y_test as an empty Series as label_positive is not in test data\n\nif len(excluded_features)>0:\n    X_train = X_train0.drop(excluded_features, axis=1, errors='ignore').copy()\n# Align columns - crucial for XGBoost to avoid 'feature_names mismatch'\n# Ensure X_test only has columns present in X_train\n#X_train = X_train0\nmissing_in_test_but_in_train = set(X_train.columns) - set(X_test.columns)\nfor c in missing_in_test_but_in_train:\n    X_test[c] = 0\n\nX_test = X_test[X_train.columns] # Crop columns in X_test that are not in X_train and ensure order\n\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_test shape: {X_test.shape}\")\nprint(f\"y_test shape: {y_test.shape}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_all = X_train.copy()\ndt_all['target'] = y_train\n\nf_cols = [c for c in dt_all.columns if c not in [\"target\"]]\n\n# Boolean mask of duplicated rows based only on f_cols\n#dupl = dt_all.duplicated(subset=f_cols, keep=\"first\")\n\n# Keep only duplicated rows\n#dt_all = dt_all[dupl].copy()\nX_train = dt_all[f_cols]\ny_train = dt_all['target']\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_test shape: {X_test.shape}\")\nprint(f\"y_test shape: {y_test.shape}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"XGB_PARAMS  = {\n    # fixed\n    'eval_metric': 'logloss',\n    'objective': 'binary:logistic',\n    'random_state': RANDOM_SEED,\n    'importance_type':'gain',\n    ## variable\n    'colsample_bytree': 0.8541925755064158,\n    'learning_rate': 0.03270545286179225,\n    'max_depth': 5,\n    'n_estimators': 73,\n    'reg_alpha': 0.007228681890108715,\n    'reg_lambda': 0.11046982689903566,\n    'subsample': 1.0\n}","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)\nval_log_loss = []\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train)):\n        print(f\"\\n--- Fold {fold+1}/{N_SPLITS} ---\")\n        X_tr, y_tr = X_train.iloc[train_idx], y_train.iloc[train_idx]\n        X_va, y_va = X_train.iloc[val_idx], y_train.iloc[val_idx]\n    \n        #\n    \n        # --- XGBoost ---\n        print(\"Training XGBoost...\")\n        # Note: XGBoost early stopping uses eval_set parameter directly in fit\n        xgb_model = xgb.XGBClassifier(**XGB_PARAMS, use_label_encoder=False)\n        xgb_model.fit(X_tr, y_tr,\n                      eval_set=[(X_va, y_va)],\n                      verbose=False) # verbose=False keeps output clean\n        #best_iter_xgb = xgb_model.best_iteration if hasattr(xgb_model, 'best_iteration') else XGB_N_ESTIMATORS # Get best iteration if early stopping triggered\n        val_preds_xgb = xgb_model.predict_proba(X_va)[:, 1]\n        val_log_loss_iter = log_loss(y_va, val_preds_xgb)\n        val_auc_iter = roc_auc_score(y_va, val_preds_xgb)\n        val_log_loss.append(val_log_loss_iter)\n        print(f\"LogLoss = {val_log_loss_iter}\")\n        print(f\"AUC = {val_auc_iter}\")\n\nprint(f\"Mean LofLoss ={np.mean(val_log_loss)} \")\nprint(f\"Median LofLoss ={np.median(val_log_loss)} \")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LGBM_PARAMS = {\n    \"device\": \"cpu\",\n    \"n_jobs\": -1,\n    \"random_state\": RANDOM_SEED,\n    \"verbose\":-1,\n    'objective': 'binary',\n    \"n_estimators\":100,\n    \"num_leaves\":31,\n    \"max_depth\":-1,\n    \"learning_rate\": 0.1,\n    'min_child_weight':0.001,\n    \"subsample\":1.0,\n    'colsample_bytree':1.0,\n    \"reg_alpha\":1e-3, \n    \"reg_lambda\":1e-3,\n}","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## LGBM\nkf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)\nval_log_loss = []\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train)):\n        print(f\"\\n--- Fold {fold+1}/{N_SPLITS} ---\")\n        X_tr, y_tr = X_train.iloc[train_idx], y_train.iloc[train_idx]\n        X_va, y_va = X_train.iloc[val_idx], y_train.iloc[val_idx]\n    \n        #\n    \n       \n        print(\"Training LGMBoost...\")\n \n        # Note: XGBoost early stopping uses eval_set parameter directly in fit\n        lgbm_model = lgb.LGBMClassifier(**LGBM_PARAMS)\n        lgbm_model.fit(X_tr, y_tr,\n                      eval_set=[(X_va, y_va)]) # verbose=False keeps output clean\n        #best_iter_xgb = xgb_model.best_iteration if hasattr(xgb_model, 'best_iteration') else XGB_N_ESTIMATORS # Get best iteration if early stopping triggered\n        val_preds_lgbm = lgbm_model.predict_proba(X_va)[:, 1]\n        val_log_loss_iter = log_loss(y_va, val_preds_lgbm)\n        val_auc_iter = roc_auc_score(y_va, val_preds_lgbm)\n        val_log_loss.append(val_log_loss_iter)\n        print(f\"LogLoss = {val_log_loss_iter}\")\n        print(f\"AUC = {val_auc_iter}\")\n\nprint(f\"Mean LofLoss ={np.mean(val_log_loss)} \")\nprint(f\"Median LofLoss ={np.median(val_log_loss)} \")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_important = xgb_model.get_booster().get_score(importance_type='weight')\nkeys = list(feature_important.keys())\nvalues = list(feature_important.values())\n\nfeature_important_df = pd.DataFrame(data=values, index=keys, columns=[\"score\"]).sort_values(by = \"score\", ascending=False)\nfeature_important_df.nlargest(20, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot top 20 features","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_important_df.nsmallest(20, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot worst 20 features","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize XGBoost Classifier\nmodel_xgb = xgb.XGBClassifier(**XGB_PARAMS)\nmodel_lgb = lgb.LGBMClassifier(**LGBM_PARAMS)\n\n# Train the model with PCA-transformed data\nprint(\"\\nTraining XGBoost model...\")\nmodel_xgb.fit(X_train, y_train)\nmodel_lgb.fit(X_train, y_train)\nprint(\"Model training complete.\")\n\n# Make predictions on the test set with PCA-transformed data\nprint(\"\\nMaking predictions on the test set...\")\ny_pred_proba_xgb = model_xgb.predict_proba(X_test)[:, 1] # Get probabilities for the positive class\ny_pred_proba_lgb = model_lgb.predict_proba(X_test)[:, 1] # Get probabilities for the positive class\ny_pred_proba = y_pred_proba_xgb*0.8 + y_pred_proba_lgb*0.2\npredictions_df = pd.DataFrame({\n    'ID': dataset_test['ID'],\n    'dataset': dataset_test['dataset'],\n    'label_positive_pred': y_pred_proba, # Store probabilities here\n})\nprint(f\"Prediction has {len(predictions_df)} rows\")\ndisplay(predictions_df.head())","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.453Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Submission**","metadata":{}},{"cell_type":"code","source":"# Construct the path to the sample submission file\nsample_submission_path = os.path.join(PATH_DATASET, 'sample_submissions.csv')\n\n# Load the sample submission file\nsample_submission_df = pd.read_csv(sample_submission_path)\nprint(f\"Sample submissions has {len(sample_submission_df)} rows\")\nprint(\"Sample Submission DataFrame Head:\")\ndisplay(sample_submission_df.head())\n\n# Merge predictions_df with sample_submission_df\n# First, drop the existing 'label_positive' from sample_submission_df if it exists, as we will replace it\nsample_submission_df = sample_submission_df.drop(columns=['label_positive_probability'])\n\n# Now merge the predictions_df, which contains 'filename' and 'label_positive_pred'\nsubmission_df = pd.merge(\n    sample_submission_df,\n    predictions_df,\n    on=['ID', 'dataset'],\n    how='left',\n)\n\n# Rename 'label_positive_pred' to 'label_positive' for the final submission format\nsubmission_df = submission_df.rename(columns={'label_positive_pred': 'label_positive_probability'})\nsubmission_df = submission_df.fillna(0.5)\n# Remove the duplicates from the final result, keeping the first occurrence\nsubmission_df = submission_df.drop_duplicates(subset=['ID', 'dataset'], keep='first')\n\nprint(f\"Sample submissions has {len(submission_df)} rows\")\n# Display the head of the final submission DataFrame\nprint(\"Final Submission DataFrame Head:\")\ndisplay(submission_df.head())\n\n# You can save this to a CSV file if needed\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-14T12:12:43.453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head 'submission.csv'","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}