{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":106680,"databundleVersionId":13374319,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":37908.409224,"end_time":"2025-11-27T00:30:29.642314","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-11-26T13:58:41.233090","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"29a6e730","cell_type":"markdown","source":"# AIRR-ML🧬25: baseline with XGBoost","metadata":{"papermill":{"duration":0.008155,"end_time":"2025-11-26T13:58:48.411872","exception":false,"start_time":"2025-11-26T13:58:48.403717","status":"completed"},"tags":[]}},{"id":"782679ef","cell_type":"code","source":"import os\nimport shutil\nimport seaborn as sns\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot\nfrom typing import List, Optional\nimport numpy as np\n\nimport xgboost as xgb\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.decomposition import PCA # Import PCA\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import roc_auc_score\nfrom xgboost import plot_importance\n\nimport optuna\n","metadata":{"execution":{"iopub.execute_input":"2025-11-26T13:58:56.319602Z","iopub.status.busy":"2025-11-26T13:58:56.319257Z","iopub.status.idle":"2025-11-26T13:59:11.004028Z","shell.execute_reply":"2025-11-26T13:59:11.002971Z"},"papermill":{"duration":14.696086,"end_time":"2025-11-26T13:59:11.006123","exception":false,"start_time":"2025-11-26T13:58:56.310037","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5ddbc7da","cell_type":"code","source":"PATH_DATASET = \"/kaggle/input/adaptive-immune-profiling-challenge-2025\"\nPATH_TRAIN_DATASETS = os.path.join(PATH_DATASET, 'train_datasets', 'train_datasets')\ntrain_datasets = sorted(os.listdir(PATH_TRAIN_DATASETS))\nprint(train_datasets)\nPATH_TEST_DATASETS = os.path.join(PATH_DATASET, 'test_datasets', 'test_datasets')\ntest_datasets = sorted(os.listdir(PATH_TEST_DATASETS))\nprint(test_datasets)","metadata":{"execution":{"iopub.execute_input":"2025-11-26T13:59:11.023404Z","iopub.status.busy":"2025-11-26T13:59:11.022741Z","iopub.status.idle":"2025-11-26T13:59:11.048058Z","shell.execute_reply":"2025-11-26T13:59:11.046923Z"},"papermill":{"duration":0.036304,"end_time":"2025-11-26T13:59:11.050231","exception":false,"start_time":"2025-11-26T13:59:11.013927","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"be2ae639","cell_type":"code","source":"RANDOM_SEED = 42\nN_SPLITS = 5","metadata":{"execution":{"iopub.execute_input":"2025-11-26T13:59:11.067010Z","iopub.status.busy":"2025-11-26T13:59:11.066353Z","iopub.status.idle":"2025-11-26T13:59:11.070954Z","shell.execute_reply":"2025-11-26T13:59:11.070038Z"},"papermill":{"duration":0.014581,"end_time":"2025-11-26T13:59:11.072584","exception":false,"start_time":"2025-11-26T13:59:11.058003","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"26f0bc7c","cell_type":"code","source":"\ndef parse_tsv_files(folder_path: str, feature_colums = ('v_call', 'j_call')):\n    folder = os.path.basename(folder_path) # Derive folder name from folder_path\n    # List all files in the directory\n    files = os.listdir(folder_path)\n\n    # Filter for .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    # Iterate through each TSV file, load it into a DataFrame, and print column names\n    dataset = []\n    for tsv_file in tqdm(tsv_files, desc=\"Loading TSV files\"):\n        file_path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n        try:\n            df = pd.read_csv(file_path, sep='\\t')\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n        \n        one_case = {\"ID\": file_name, \"dataset\": folder}\n        if metadata is not None:\n            one_case = {\"label_positive\": int(metadata.at[tsv_file, \"label_positive\"])}\n        for col in feature_colums:\n            counts = df[col].value_counts() / len(df)\n            one_case.update(counts.to_dict())\n            # print(one_case)\n            dataset.append(one_case)\n\n    return dataset","metadata":{"execution":{"iopub.execute_input":"2025-11-26T13:59:11.089643Z","iopub.status.busy":"2025-11-26T13:59:11.089319Z","iopub.status.idle":"2025-11-26T13:59:11.099174Z","shell.execute_reply":"2025-11-26T13:59:11.098144Z"},"papermill":{"duration":0.020523,"end_time":"2025-11-26T13:59:11.101088","exception":false,"start_time":"2025-11-26T13:59:11.080565","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c54b2ec0","cell_type":"markdown","source":"## Train dataset","metadata":{"papermill":{"duration":0.007031,"end_time":"2025-11-26T13:59:11.115429","exception":false,"start_time":"2025-11-26T13:59:11.108398","status":"completed"},"tags":[]}},{"id":"bbf90aec","cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm import tqdm\nfrom collections import Counter\n\n\ndef extract_kmers(seq: str, k: int = 5):\n    \"\"\"Return all k-mers from an amino-acid sequence.\"\"\"\n    if not isinstance(seq, str):\n        return []\n    seq = seq.strip()\n    if len(seq) < k:\n        return []\n    return [seq[i:i+k] for i in range(len(seq) - k + 1)]\n\n\ndef parse_tsv_files(\n        folder_path: str, \n        feature_columns=('v_call', 'j_call'),\n        kmer_column='junction_aa',\n        k=3  # length of kmers\n):\n    folder = os.path.basename(folder_path)\n    files = os.listdir(folder_path)\n\n    # Only .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    # Metadata\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    dataset = []\n\n    for tsv_file in tqdm(tsv_files):\n        path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n\n        try:\n            df = pd.read_csv(path, sep=\"\\t\")\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n            continue\n\n        # Collect all features here\n        one_case = {\"ID\": file_name, \"dataset\": folder}\n\n        if metadata is not None:\n            one_case[\"label_positive\"] = int(metadata.at[tsv_file, \"label_positive\"])\n\n        # === 1) Count V / J genes ===\n        for col in feature_columns:\n            counts = df[col].value_counts(normalize=True)  # normalized frequency %\n            one_case.update({f\"{col}__{k}\": v for k, v in counts.to_dict().items()})\n\n        # === 2) Count k-mers from junction_aa ===\n        if kmer_column in df.columns:\n            all_kmers = []\n\n            for seq in df[kmer_column].dropna():\n                all_kmers.extend(extract_kmers(seq, k=k))\n\n            kmer_counts = Counter(all_kmers)\n\n            # convert to relative frequencies\n            total = sum(kmer_counts.values())\n            if total > 0:\n                kmer_freqs = {f\"kmer{k}_{mer}\": count / total for mer, count in kmer_counts.items()}\n            else:\n                kmer_freqs = {}\n\n            one_case.update(kmer_freqs)\n\n        dataset.append(one_case)\n\n    return dataset\n","metadata":{"execution":{"iopub.execute_input":"2025-11-26T13:59:11.132443Z","iopub.status.busy":"2025-11-26T13:59:11.131294Z","iopub.status.idle":"2025-11-26T13:59:11.146606Z","shell.execute_reply":"2025-11-26T13:59:11.145572Z"},"papermill":{"duration":0.025878,"end_time":"2025-11-26T13:59:11.148488","exception":false,"start_time":"2025-11-26T13:59:11.122610","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1c2fe51c","cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(train_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TRAIN_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_train = pd.DataFrame(dataset)\n#display(dataset_train)","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2025-11-26T13:59:11.165062Z","iopub.status.busy":"2025-11-26T13:59:11.164659Z","iopub.status.idle":"2025-11-26T14:25:46.640792Z","shell.execute_reply":"2025-11-26T14:25:46.639666Z"},"papermill":{"duration":1595.487125,"end_time":"2025-11-26T14:25:46.642704","exception":false,"start_time":"2025-11-26T13:59:11.155579","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"15fdc7ae","cell_type":"code","source":"dataset_train.head()\n","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:25:47.163631Z","iopub.status.busy":"2025-11-26T14:25:47.163281Z","iopub.status.idle":"2025-11-26T14:25:47.225148Z","shell.execute_reply":"2025-11-26T14:25:47.223725Z"},"papermill":{"duration":0.374312,"end_time":"2025-11-26T14:25:47.227149","exception":false,"start_time":"2025-11-26T14:25:46.852837","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d22ba3d0","cell_type":"code","source":"f_cols = [col for col in dataset_train.columns if col not in ['ID', 'dataset']]\n\ncols_na = dataset_train[f_cols].isnull().sum()\n\ncols_na = cols_na[cols_na > 3100]\nexcluded_features = cols_na.index.to_list()","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:25:47.646440Z","iopub.status.busy":"2025-11-26T14:25:47.646126Z","iopub.status.idle":"2025-11-26T14:25:47.867364Z","shell.execute_reply":"2025-11-26T14:25:47.866073Z"},"papermill":{"duration":0.432265,"end_time":"2025-11-26T14:25:47.869290","exception":false,"start_time":"2025-11-26T14:25:47.437025","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3463fd5a","cell_type":"code","source":"len(excluded_features)","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:25:48.298623Z","iopub.status.busy":"2025-11-26T14:25:48.297606Z","iopub.status.idle":"2025-11-26T14:25:48.306285Z","shell.execute_reply":"2025-11-26T14:25:48.305017Z"},"papermill":{"duration":0.226037,"end_time":"2025-11-26T14:25:48.308550","exception":false,"start_time":"2025-11-26T14:25:48.082513","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e97c727a","cell_type":"markdown","source":"## Test dataset","metadata":{"papermill":{"duration":0.209006,"end_time":"2025-11-26T14:25:48.739986","exception":false,"start_time":"2025-11-26T14:25:48.530980","status":"completed"},"tags":[]}},{"id":"ef5b0a54","cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(test_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TEST_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_test = pd.DataFrame(dataset)\n#display(dataset_test)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2025-11-26T14:25:49.162004Z","iopub.status.busy":"2025-11-26T14:25:49.161263Z","iopub.status.idle":"2025-11-26T14:48:58.005564Z","shell.execute_reply":"2025-11-26T14:48:58.003475Z"},"papermill":{"duration":1389.054344,"end_time":"2025-11-26T14:48:58.008207","exception":false,"start_time":"2025-11-26T14:25:48.953863","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b4c193cb","cell_type":"code","source":"f_cols = [col for col in dataset_test.columns if col not in ['ID', 'dataset']]\n\n# 2. Sum of missing values\ncols_na = dataset_test[f_cols].isnull().sum()\n\n# 3. Just an attempt to reduce variables with many missing values\ncols_na = cols_na[cols_na > 4000]\nexcluded_features_test = cols_na.index.to_list()\n\nexcluded_features =  excluded_features  +  excluded_features_test\nexcluded_features = list(set(excluded_features))\nlen(excluded_features)","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:48:59.041347Z","iopub.status.busy":"2025-11-26T14:48:59.040609Z","iopub.status.idle":"2025-11-26T14:48:59.327397Z","shell.execute_reply":"2025-11-26T14:48:59.326338Z"},"papermill":{"duration":0.844293,"end_time":"2025-11-26T14:48:59.329277","exception":false,"start_time":"2025-11-26T14:48:58.484984","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2d9b2321","cell_type":"code","source":"\nextra_excluded = [\"TCRBV22.X\",\"TCRBV29.X\",\"TCRBV25.or09_02\", \"TCRBV29.or09_02\", \n                      \"X\",\"TCRBV24\"]\nexcluded_features +=  extra_excluded\nexcluded_features = list(set(excluded_features))\nprint(f\"n_excluded: {len(excluded_features)}\")\nexcluded_features[:10]","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:49:00.216269Z","iopub.status.busy":"2025-11-26T14:49:00.215504Z","iopub.status.idle":"2025-11-26T14:49:00.225027Z","shell.execute_reply":"2025-11-26T14:49:00.224099Z"},"papermill":{"duration":0.45445,"end_time":"2025-11-26T14:49:00.226567","exception":false,"start_time":"2025-11-26T14:48:59.772117","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8af5c8cb","cell_type":"markdown","source":"## Training XGBoost","metadata":{"papermill":{"duration":0.442809,"end_time":"2025-11-26T14:49:01.109171","exception":false,"start_time":"2025-11-26T14:49:00.666362","status":"completed"},"tags":[]}},{"id":"b6ff83cb","cell_type":"code","source":"\n\n# Prepare the data\nX_train0 = dataset_train.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_train = dataset_train['label_positive']\n\nX_test = dataset_test.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_test = pd.Series([]) # Initialize y_test as an empty Series as label_positive is not in test data\n\nif len(excluded_features)>0:\n    X_train = X_train0.drop(excluded_features, axis=1, errors='ignore').copy()\n# Align columns - crucial for XGBoost to avoid 'feature_names mismatch'\n# Ensure X_test only has columns present in X_train\n#X_train = X_train0\nmissing_in_test_but_in_train = set(X_train.columns) - set(X_test.columns)\nfor c in missing_in_test_but_in_train:\n    X_test[c] = 0\n\nX_test = X_test[X_train.columns] # Crop columns in X_test that are not in X_train and ensure order\n\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_test shape: {X_test.shape}\")\nprint(f\"y_test shape: {y_test.shape}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:49:03.009623Z","iopub.status.busy":"2025-11-26T14:49:03.009304Z","iopub.status.idle":"2025-11-26T14:49:04.418968Z","shell.execute_reply":"2025-11-26T14:49:04.417621Z"},"papermill":{"duration":1.855498,"end_time":"2025-11-26T14:49:04.420725","exception":false,"start_time":"2025-11-26T14:49:02.565227","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4732a1a9","cell_type":"code","source":"\ndt_all = X_train.copy()\ndt_all['target'] = y_train\n\nf_cols = [c for c in dt_all.columns if c not in [\"target\"]]\n\n# Boolean mask of duplicated rows based only on f_cols\n#dupl = dt_all.duplicated(subset=f_cols, keep=\"first\")\n\n# Keep only duplicated rows\n#dt_all = dt_all[dupl].copy()\nX_train = dt_all[f_cols]\ny_train = dt_all['target']\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_test shape: {X_test.shape}\")\nprint(f\"y_test shape: {y_test.shape}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:49:05.305653Z","iopub.status.busy":"2025-11-26T14:49:05.305286Z","iopub.status.idle":"2025-11-26T14:49:05.473090Z","shell.execute_reply":"2025-11-26T14:49:05.471697Z"},"papermill":{"duration":0.612281,"end_time":"2025-11-26T14:49:05.475263","exception":false,"start_time":"2025-11-26T14:49:04.862982","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"6d0ae0f9","cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:49:06.467441Z","iopub.status.busy":"2025-11-26T14:49:06.466057Z","iopub.status.idle":"2025-11-26T14:49:06.497845Z","shell.execute_reply":"2025-11-26T14:49:06.496565Z"},"papermill":{"duration":0.477378,"end_time":"2025-11-26T14:49:06.499622","exception":false,"start_time":"2025-11-26T14:49:06.022244","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"581911d1","cell_type":"code","source":"XGB_PARAMS  = {\n    # fixed\n    'eval_metric': 'logloss',\n    'objective': 'binary:logistic',\n    'random_state': RANDOM_SEED,\n    'importance_type':'gain',\n    ## variable\n    'colsample_bytree': 0.8541925755064158,\n    'learning_rate': 0.03270545286179225,\n    'max_depth': 5,\n    'n_estimators': 73,\n    'reg_alpha': 0.007228681890108715,\n    'reg_lambda': 0.11046982689903566,\n    'subsample': 1.0\n}\n","metadata":{"execution":{"iopub.execute_input":"2025-11-26T14:49:08.293369Z","iopub.status.busy":"2025-11-26T14:49:08.292570Z","iopub.status.idle":"2025-11-26T14:49:08.300862Z","shell.execute_reply":"2025-11-26T14:49:08.299419Z"},"papermill":{"duration":0.463713,"end_time":"2025-11-26T14:49:08.303463","exception":false,"start_time":"2025-11-26T14:49:07.839750","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8ccff714","cell_type":"code","source":"kf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)\nval_log_loss = []\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train)):\n        print(f\"\\n--- Fold {fold+1}/{N_SPLITS} ---\")\n        X_tr, y_tr = X_train.iloc[train_idx], y_train.iloc[train_idx]\n        X_va, y_va = X_train.iloc[val_idx], y_train.iloc[val_idx]\n    \n        #\n    \n        # --- XGBoost ---\n        print(\"Training XGBoost...\")\n        # Note: XGBoost early stopping uses eval_set parameter directly in fit\n        xgb_model = xgb.XGBClassifier(**XGB_PARAMS, use_label_encoder=False)\n        xgb_model.fit(X_tr, y_tr,\n                      eval_set=[(X_va, y_va)],\n                      verbose=False) # verbose=False keeps output clean\n        #best_iter_xgb = xgb_model.best_iteration if hasattr(xgb_model, 'best_iteration') else XGB_N_ESTIMATORS # Get best iteration if early stopping triggered\n        val_preds_xgb = xgb_model.predict_proba(X_va)[:, 1]\n        val_log_loss_iter = log_loss(y_va, val_preds_xgb)\n        val_auc_iter = roc_auc_score(y_va, val_preds_xgb)\n        val_log_loss.append(val_log_loss_iter)\n        print(f\"LogLoss = {val_log_loss_iter}\")\n        print(f\"AUC = {val_auc_iter}\")\n\nprint(f\"Mean LofLoss ={np.mean(val_log_loss)} \")\nprint(f\"Median LofLoss ={np.median(val_log_loss)} \")","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:15:09.639156Z","iopub.status.busy":"2025-11-27T00:15:09.638416Z","iopub.status.idle":"2025-11-27T00:21:16.686717Z","shell.execute_reply":"2025-11-27T00:21:16.685586Z"},"papermill":{"duration":368.059399,"end_time":"2025-11-27T00:21:17.247933","exception":false,"start_time":"2025-11-27T00:15:09.188534","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"50d5236a","cell_type":"code","source":"LGBM_PARAMS = {\n    \"device\": \"cpu\",\n    \"n_jobs\": -1,\n    \"random_state\": RANDOM_SEED,\n    \"verbose\":-1,\n    'objective': 'binary',\n    \"n_estimators\":100,\n    \"num_leaves\":31,\n    \"max_depth\":-1,\n    \"learning_rate\": 0.1,\n    'min_child_weight':0.001,\n    \"subsample\":1.0,\n    'colsample_bytree':1.0,\n    \"reg_alpha\":1e-3, \n    \"reg_lambda\":1e-3,\n}\n\n","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:21:18.137976Z","iopub.status.busy":"2025-11-27T00:21:18.137494Z","iopub.status.idle":"2025-11-27T00:21:18.144556Z","shell.execute_reply":"2025-11-27T00:21:18.143507Z"},"papermill":{"duration":0.455897,"end_time":"2025-11-27T00:21:18.146140","exception":false,"start_time":"2025-11-27T00:21:17.690243","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"9f3108df","cell_type":"code","source":"## LGBM\nkf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)\nval_log_loss = []\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y_train)):\n        print(f\"\\n--- Fold {fold+1}/{N_SPLITS} ---\")\n        X_tr, y_tr = X_train.iloc[train_idx], y_train.iloc[train_idx]\n        X_va, y_va = X_train.iloc[val_idx], y_train.iloc[val_idx]\n    \n        #\n    \n       \n        print(\"Training LGMBoost...\")\n \n        # Note: XGBoost early stopping uses eval_set parameter directly in fit\n        lgbm_model = lgb.LGBMClassifier(**LGBM_PARAMS)\n        lgbm_model.fit(X_tr, y_tr,\n                      eval_set=[(X_va, y_va)]) # verbose=False keeps output clean\n        #best_iter_xgb = xgb_model.best_iteration if hasattr(xgb_model, 'best_iteration') else XGB_N_ESTIMATORS # Get best iteration if early stopping triggered\n        val_preds_lgbm = lgbm_model.predict_proba(X_va)[:, 1]\n        val_log_loss_iter = log_loss(y_va, val_preds_lgbm)\n        val_auc_iter = roc_auc_score(y_va, val_preds_lgbm)\n        val_log_loss.append(val_log_loss_iter)\n        print(f\"LogLoss = {val_log_loss_iter}\")\n        print(f\"AUC = {val_auc_iter}\")\n\nprint(f\"Mean LofLoss ={np.mean(val_log_loss)} \")\nprint(f\"Median LofLoss ={np.median(val_log_loss)} \")","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:21:20.949018Z","iopub.status.busy":"2025-11-27T00:21:20.948536Z","iopub.status.idle":"2025-11-27T00:27:49.375742Z","shell.execute_reply":"2025-11-27T00:27:49.374908Z"},"papermill":{"duration":389.314067,"end_time":"2025-11-27T00:27:49.818773","exception":false,"start_time":"2025-11-27T00:21:20.504706","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1b58f0f7","cell_type":"code","source":"feature_important = xgb_model.get_booster().get_score(importance_type='weight')\nkeys = list(feature_important.keys())\nvalues = list(feature_important.values())\n\nfeature_important_df = pd.DataFrame(data=values, index=keys, columns=[\"score\"]).sort_values(by = \"score\", ascending=False)\nfeature_important_df.nlargest(20, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot top 20 features","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:27:50.702516Z","iopub.status.busy":"2025-11-27T00:27:50.701166Z","iopub.status.idle":"2025-11-27T00:27:51.523865Z","shell.execute_reply":"2025-11-27T00:27:51.522854Z"},"papermill":{"duration":1.263225,"end_time":"2025-11-27T00:27:51.525489","exception":false,"start_time":"2025-11-27T00:27:50.262264","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c725a89e","cell_type":"code","source":"feature_important_df.nsmallest(20, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot worst 20 features","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:27:52.521183Z","iopub.status.busy":"2025-11-27T00:27:52.520840Z","iopub.status.idle":"2025-11-27T00:27:52.900619Z","shell.execute_reply":"2025-11-27T00:27:52.899617Z"},"papermill":{"duration":0.938298,"end_time":"2025-11-27T00:27:52.902761","exception":false,"start_time":"2025-11-27T00:27:51.964463","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"22a75cd0","cell_type":"code","source":"\n# Initialize XGBoost Classifier\nmodel_xgb = xgb.XGBClassifier(**XGB_PARAMS)\nmodel_lgb = lgb.LGBMClassifier(**LGBM_PARAMS)\n\n# Train the model with PCA-transformed data\nprint(\"\\nTraining XGBoost model...\")\nmodel_xgb.fit(X_train, y_train)\nmodel_lgb.fit(X_train, y_train)\nprint(\"Model training complete.\")\n\n# Make predictions on the test set with PCA-transformed data\nprint(\"\\nMaking predictions on the test set...\")\ny_pred_proba_xgb = model_xgb.predict_proba(X_test)[:, 1] # Get probabilities for the positive class\ny_pred_proba_lgb = model_lgb.predict_proba(X_test)[:, 1] # Get probabilities for the positive class\ny_pred_proba = y_pred_proba_xgb*0.8 + y_pred_proba_lgb*0.2\npredictions_df = pd.DataFrame({\n    'ID': dataset_test['ID'],\n    'dataset': dataset_test['dataset'],\n    'label_positive_pred': y_pred_proba, # Store probabilities here\n})\nprint(f\"Prediction has {len(predictions_df)} rows\")\ndisplay(predictions_df.head())","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:27:55.020125Z","iopub.status.busy":"2025-11-27T00:27:55.019707Z","iopub.status.idle":"2025-11-27T00:30:19.605360Z","shell.execute_reply":"2025-11-27T00:30:19.604304Z"},"papermill":{"duration":145.06128,"end_time":"2025-11-27T00:30:19.607297","exception":false,"start_time":"2025-11-27T00:27:54.546017","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e029cc1a","cell_type":"markdown","source":"## Prerape submission","metadata":{"papermill":{"duration":0.44017,"end_time":"2025-11-27T00:30:20.608054","exception":false,"start_time":"2025-11-27T00:30:20.167884","status":"completed"},"tags":[]}},{"id":"ad56ee75","cell_type":"code","source":"# Construct the path to the sample submission file\nsample_submission_path = os.path.join(PATH_DATASET, 'sample_submissions.csv')\n\n# Load the sample submission file\nsample_submission_df = pd.read_csv(sample_submission_path)\nprint(f\"Sample submissions has {len(sample_submission_df)} rows\")\nprint(\"Sample Submission DataFrame Head:\")\ndisplay(sample_submission_df.head())\n\n# Merge predictions_df with sample_submission_df\n# First, drop the existing 'label_positive' from sample_submission_df if it exists, as we will replace it\nsample_submission_df = sample_submission_df.drop(columns=['label_positive_probability'])\n\n# Now merge the predictions_df, which contains 'filename' and 'label_positive_pred'\nsubmission_df = pd.merge(\n    sample_submission_df,\n    predictions_df,\n    on=['ID', 'dataset'],\n    how='left',\n)\n\n# Rename 'label_positive_pred' to 'label_positive' for the final submission format\nsubmission_df = submission_df.rename(columns={'label_positive_pred': 'label_positive_probability'})\nsubmission_df = submission_df.fillna(0.5)\n# Remove the duplicates from the final result, keeping the first occurrence\nsubmission_df = submission_df.drop_duplicates(subset=['ID', 'dataset'], keep='first')\n\nprint(f\"Sample submissions has {len(submission_df)} rows\")\n# Display the head of the final submission DataFrame\nprint(\"Final Submission DataFrame Head:\")\ndisplay(submission_df.head())\n\n# You can save this to a CSV file if needed\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:30:21.501967Z","iopub.status.busy":"2025-11-27T00:30:21.501010Z","iopub.status.idle":"2025-11-27T00:30:24.885462Z","shell.execute_reply":"2025-11-27T00:30:24.884275Z"},"papermill":{"duration":3.834655,"end_time":"2025-11-27T00:30:24.887333","exception":false,"start_time":"2025-11-27T00:30:21.052678","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"62cd4c59","cell_type":"code","source":"!head 'submission.csv'","metadata":{"execution":{"iopub.execute_input":"2025-11-27T00:30:25.774285Z","iopub.status.busy":"2025-11-27T00:30:25.773883Z","iopub.status.idle":"2025-11-27T00:30:26.165104Z","shell.execute_reply":"2025-11-27T00:30:26.163657Z"},"papermill":{"duration":0.834534,"end_time":"2025-11-27T00:30:26.167202","exception":false,"start_time":"2025-11-27T00:30:25.332668","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}