{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":106680,"databundleVersionId":13374319,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport seaborn as sns\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot\nfrom typing import List, Optional\nimport numpy as np","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:07:41.244002Z","iopub.execute_input":"2025-12-04T12:07:41.244712Z","iopub.status.idle":"2025-12-04T12:07:42.516394Z","shell.execute_reply.started":"2025-12-04T12:07:41.244681Z","shell.execute_reply":"2025-12-04T12:07:42.515460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH_DATASET = \"/kaggle/input/adaptive-immune-profiling-challenge-2025\"\nPATH_TRAIN_DATASETS = os.path.join(PATH_DATASET, 'train_datasets', 'train_datasets')\ntrain_datasets = sorted(os.listdir(PATH_TRAIN_DATASETS))\nprint(train_datasets)\nPATH_TEST_DATASETS = os.path.join(PATH_DATASET, 'test_datasets', 'test_datasets')\ntest_datasets = sorted(os.listdir(PATH_TEST_DATASETS))\nprint(test_datasets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:08:54.791883Z","iopub.execute_input":"2025-12-04T12:08:54.792384Z","iopub.status.idle":"2025-12-04T12:08:54.808778Z","shell.execute_reply.started":"2025-12-04T12:08:54.792355Z","shell.execute_reply":"2025-12-04T12:08:54.807832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train dataset","metadata":{}},{"cell_type":"code","source":"def parse_tsv_files(folder_path: str, feature_colums = ('v_call', 'j_call')):\n    folder = os.path.basename(folder_path) # Derive folder name from folder_path\n    # List all files in the directory\n    files = os.listdir(folder_path)\n\n    # Filter for .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    # Iterate through each TSV file, load it into a DataFrame, and print column names\n    dataset = []\n    for tsv_file in tqdm(tsv_files, desc=\"Loading TSV files\"):\n        file_path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n        try:\n            df = pd.read_csv(file_path, sep='\\t')\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n        \n        one_case = {\"ID\": file_name, \"dataset\": folder}\n        if metadata is not None:\n            one_case = {\"label_positive\": int(metadata.at[tsv_file, \"label_positive\"])}\n        for col in feature_colums:\n            counts = df[col].value_counts() / len(df)\n            one_case.update(counts.to_dict())\n            # print(one_case)\n            dataset.append(one_case)\n\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:10:34.875712Z","iopub.execute_input":"2025-12-04T12:10:34.876108Z","iopub.status.idle":"2025-12-04T12:10:34.884734Z","shell.execute_reply.started":"2025-12-04T12:10:34.876071Z","shell.execute_reply":"2025-12-04T12:10:34.883969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter\n\ndef extract_kmers(seq: str, k: int = 5):\n    \"\"\"Return all k-mers from an amino-acid sequence.\"\"\"\n    if not isinstance(seq, str):\n        return []\n    seq = seq.strip()\n    if len(seq) < k:\n        return []\n    return [seq[i:i+k] for i in range(len(seq) - k + 1)]\n\n\ndef parse_tsv_files(\n        folder_path: str, \n        feature_columns=('v_call', 'j_call'),\n        kmer_column='junction_aa',\n        k=3  # length of kmers\n):\n    folder = os.path.basename(folder_path)\n    files = os.listdir(folder_path)\n\n    # Only .tsv files\n    tsv_files = [f for f in files if f.endswith('.tsv')]\n    other_files = [f.name for f in os.scandir(folder_path) if not f.name.endswith('.tsv')]\n    print(f'Loading {len(tsv_files)} .tsv files from {folder} (remaining: {other_files}).')\n\n    # Metadata\n    metadata = None\n    if \"metadata.csv\" in files:\n        metadata = pd.read_csv(os.path.join(folder_path, \"metadata.csv\"))\n        metadata.set_index(\"filename\", inplace=True)\n\n    dataset = []\n\n    for tsv_file in tqdm(tsv_files):\n        path = os.path.join(folder_path, tsv_file)\n        file_name, _ = os.path.splitext(tsv_file)\n\n        try:\n            df = pd.read_csv(path, sep=\"\\t\")\n        except Exception as e:\n            print(f\"Error loading {tsv_file}: {e}\")\n            continue\n\n        # Collect all features here\n        one_case = {\"ID\": file_name, \"dataset\": folder}\n\n        if metadata is not None:\n            one_case[\"label_positive\"] = int(metadata.at[tsv_file, \"label_positive\"])\n\n        # === 1) Count V / J genes ===\n        for col in feature_columns:\n            counts = df[col].value_counts(normalize=True)  # normalized frequency %\n            one_case.update({f\"{col}__{k}\": v for k, v in counts.to_dict().items()})\n\n        # === 2) Count k-mers from junction_aa ===\n        if kmer_column in df.columns:\n            all_kmers = []\n\n            for seq in df[kmer_column].dropna():\n                all_kmers.extend(extract_kmers(seq, k=k))\n\n            kmer_counts = Counter(all_kmers)\n\n            # convert to relative frequencies\n            total = sum(kmer_counts.values())\n            if total > 0:\n                kmer_freqs = {f\"kmer{k}_{mer}\": count / total for mer, count in kmer_counts.items()}\n            else:\n                kmer_freqs = {}\n\n            one_case.update(kmer_freqs)\n\n        dataset.append(one_case)\n\n    return dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:11:49.752577Z","iopub.execute_input":"2025-12-04T12:11:49.752899Z","iopub.status.idle":"2025-12-04T12:11:49.769914Z","shell.execute_reply.started":"2025-12-04T12:11:49.752875Z","shell.execute_reply":"2025-12-04T12:11:49.769064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(train_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TRAIN_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_train = pd.DataFrame(dataset)\n#display(dataset_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:11:53.963630Z","iopub.execute_input":"2025-12-04T12:11:53.963950Z","iopub.status.idle":"2025-12-04T12:35:00.202797Z","shell.execute_reply.started":"2025-12-04T12:11:53.963925Z","shell.execute_reply":"2025-12-04T12:35:00.200786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:35:03.766740Z","iopub.execute_input":"2025-12-04T12:35:03.767174Z","iopub.status.idle":"2025-12-04T12:35:03.817072Z","shell.execute_reply.started":"2025-12-04T12:35:03.767123Z","shell.execute_reply":"2025-12-04T12:35:03.816221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f_cols = [col for col in dataset_train.columns if col not in ['ID', 'dataset']]\n\ncols_na = dataset_train[f_cols].isnull().sum()\n\ncols_na = cols_na[cols_na > 3100]\nexcluded_features = cols_na.index.to_list()\n\nlen(excluded_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:35:18.178081Z","iopub.execute_input":"2025-12-04T12:35:18.178437Z","iopub.status.idle":"2025-12-04T12:35:18.417757Z","shell.execute_reply.started":"2025-12-04T12:35:18.178415Z","shell.execute_reply":"2025-12-04T12:35:18.417106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test dataset","metadata":{}},{"cell_type":"code","source":"# Iterate over all sub-datasets\ndataset = []\nfor folder in tqdm(test_datasets, disable=True):\n    path_dataset_ = os.path.join(PATH_TEST_DATASETS, folder)\n    dataset += parse_tsv_files(path_dataset_)\n\ndataset_test = pd.DataFrame(dataset)\n#display(dataset_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T12:35:21.778262Z","iopub.execute_input":"2025-12-04T12:35:21.778588Z","iopub.status.idle":"2025-12-04T12:56:34.256265Z","shell.execute_reply.started":"2025-12-04T12:35:21.778565Z","shell.execute_reply":"2025-12-04T12:56:34.254772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f_cols = [col for col in dataset_test.columns if col not in ['ID', 'dataset']]\n\n# 2. Sum of missing values\ncols_na = dataset_test[f_cols].isnull().sum()\n\n# 3. Just an attempt to reduce variables with many missing values\ncols_na = cols_na[cols_na > 4000]\nexcluded_features_test = cols_na.index.to_list()\n\nexcluded_features =  excluded_features  +  excluded_features_test\nexcluded_features = list(set(excluded_features))\nlen(excluded_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:03:26.818398Z","iopub.execute_input":"2025-12-04T13:03:26.820099Z","iopub.status.idle":"2025-12-04T13:03:27.122179Z","shell.execute_reply.started":"2025-12-04T13:03:26.820054Z","shell.execute_reply":"2025-12-04T13:03:27.121122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"extra_excluded = [\"TCRBV22.X\",\"TCRBV29.X\",\"TCRBV25.or09_02\", \"TCRBV29.or09_02\", \n                      \"X\",\"TCRBV24\"]\nexcluded_features +=  extra_excluded\nexcluded_features = list(set(excluded_features))\nprint(f\"n_excluded: {len(excluded_features)}\")\nexcluded_features[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:03:30.347785Z","iopub.execute_input":"2025-12-04T13:03:30.348107Z","iopub.status.idle":"2025-12-04T13:03:30.356686Z","shell.execute_reply.started":"2025-12-04T13:03:30.348084Z","shell.execute_reply":"2025-12-04T13:03:30.355590Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare the data for training","metadata":{}},{"cell_type":"code","source":"X_train0 = dataset_train.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_train = dataset_train['label_positive']\n\nX_test = dataset_test.drop(\n    ['label_positive', 'ID', 'dataset'], axis=1, errors='ignore').fillna(0)\ny_test = pd.Series([]) # Initialize y_test as an empty Series as label_positive is not in test data\n\nif len(excluded_features)>0:\n    X_train = X_train0.drop(excluded_features, axis=1, errors='ignore').copy()\n# Align columns - crucial for XGBoost to avoid 'feature_names mismatch'\n# Ensure X_test only has columns present in X_train\n#X_train = X_train0\nmissing_in_test_but_in_train = set(X_train.columns) - set(X_test.columns)\nfor c in missing_in_test_but_in_train:\n    X_test[c] = 0\n\nX_test = X_test[X_train.columns] # Crop columns in X_test that are not in X_train and ensure order\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_test shape: {X_test.shape}\")\nprint(f\"y_test shape: {y_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:03:37.267979Z","iopub.execute_input":"2025-12-04T13:03:37.268754Z","iopub.status.idle":"2025-12-04T13:03:38.507779Z","shell.execute_reply.started":"2025-12-04T13:03:37.268725Z","shell.execute_reply":"2025-12-04T13:03:38.506968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Apply PCA after column alignment","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import StackingClassifier, RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\n\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\n\n\n# ------------------- Scale the data (required for PCA) -------------------\nscaler = StandardScaler()\n\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled  = scaler.transform(X_test)\n\n# ------------------- PCA (keep 95% variance or set fixed components) -------------------\npca = PCA(n_components=0.95, random_state=42)   # or: n_components=300\nX_train_pca = pca.fit_transform(X_train_scaled)\nX_test_pca  = pca.transform(X_test_scaled)\n\n\nprint(\"PCA components:\", X_train_pca.shape[1])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:20:46.050256Z","iopub.execute_input":"2025-12-04T13:20:46.050959Z","iopub.status.idle":"2025-12-04T13:21:25.371047Z","shell.execute_reply.started":"2025-12-04T13:20:46.050929Z","shell.execute_reply":"2025-12-04T13:21:25.370287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_tr, X_val, y_tr, y_val = train_test_split(\n    X_train_pca, y_train, test_size=0.2,\n    random_state=42, stratify=y_train\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:21:25.372360Z","iopub.execute_input":"2025-12-04T13:21:25.373570Z","iopub.status.idle":"2025-12-04T13:21:25.462021Z","shell.execute_reply.started":"2025-12-04T13:21:25.373542Z","shell.execute_reply":"2025-12-04T13:21:25.461338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Individual Models\n","metadata":{}},{"cell_type":"code","source":"base_models = [\n    (\"xgb\", xgb.XGBClassifier(\n        n_estimators=700, learning_rate=0.05, max_depth=5,\n        subsample=0.9, colsample_bytree=0.9, eval_metric='logloss'\n    )),\n\n    (\"lgb\", lgb.LGBMClassifier(\n        n_estimators=700, learning_rate=0.03,\n        num_leaves=31, subsample=0.8, colsample_bytree=0.8, verbosity=-1\n    )),\n\n    (\"cat\", CatBoostClassifier(\n        iterations=600,\n        learning_rate=0.05,\n        depth=6,\n        loss_function='Logloss',\n        verbose=False\n    )),\n\n]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:22:12.571289Z","iopub.execute_input":"2025-12-04T13:22:12.572042Z","iopub.status.idle":"2025-12-04T13:22:12.580595Z","shell.execute_reply.started":"2025-12-04T13:22:12.572014Z","shell.execute_reply":"2025-12-04T13:22:12.579742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Meta-Model (Final Estimator)","metadata":{}},{"cell_type":"code","source":"meta_model = CatBoostClassifier(\n    iterations=500,\n    depth=6,\n    learning_rate=0.03,\n    loss_function='Logloss',\n    verbose=False\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:23:17.810476Z","iopub.execute_input":"2025-12-04T13:23:17.811234Z","iopub.status.idle":"2025-12-04T13:23:17.815624Z","shell.execute_reply.started":"2025-12-04T13:23:17.811203Z","shell.execute_reply":"2025-12-04T13:23:17.814722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build the Stacking Classifier","metadata":{}},{"cell_type":"code","source":"stack_model = StackingClassifier(\n    estimators=base_models,\n    final_estimator=meta_model,\n    stack_method='predict_proba',\n    passthrough=True,\n    n_jobs=-1\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:24:08.508629Z","iopub.execute_input":"2025-12-04T13:24:08.508952Z","iopub.status.idle":"2025-12-04T13:24:08.514261Z","shell.execute_reply.started":"2025-12-04T13:24:08.508927Z","shell.execute_reply":"2025-12-04T13:24:08.513341Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train & Evaluate","metadata":{}},{"cell_type":"code","source":"stack_model.fit(X_tr, y_tr)\n\nval_pred = stack_model.predict_proba(X_val)[:, 1]\nprint(\"Stacking + CatBoost AUC:\", roc_auc_score(y_val, val_pred))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T13:24:40.935757Z","iopub.execute_input":"2025-12-04T13:24:40.936096Z","iopub.status.idle":"2025-12-04T14:02:05.118669Z","shell.execute_reply.started":"2025-12-04T13:24:40.936072Z","shell.execute_reply":"2025-12-04T14:02:05.116848Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Retrain full X_train","metadata":{}},{"cell_type":"code","source":"stack_model.fit(X_train_pca, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T14:02:05.122372Z","iopub.execute_input":"2025-12-04T14:02:05.122900Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Predict on Test Set","metadata":{}},{"cell_type":"code","source":"test_pred = stack_model.predict_proba(X_test_pca)[:, 1]\nprint(\"Test prediction shape:\", test_pred.shape)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions_df = pd.DataFrame({\n    'ID': dataset_test['ID'],\n    'dataset': dataset_test['dataset'],\n    'label_positive_pred': test_pred, # Store probabilities here\n})\nprint(f\"Prediction has {len(predictions_df)} rows\")\ndisplay(predictions_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare submission","metadata":{}},{"cell_type":"code","source":"# Construct the path to the sample submission file\nsample_submission_path = os.path.join(PATH_DATASET, 'sample_submissions.csv')\n\n# Load the sample submission file\nsample_submission_df = pd.read_csv(sample_submission_path)\nprint(f\"Sample submissions has {len(sample_submission_df)} rows\")\nprint(\"Sample Submission DataFrame Head:\")\ndisplay(sample_submission_df.head())\n\n# Merge predictions_df with sample_submission_df\n# First, drop the existing 'label_positive' from sample_submission_df if it exists, as we will replace it\nsample_submission_df = sample_submission_df.drop(columns=['label_positive_probability'])\n\n# Now merge the predictions_df, which contains 'filename' and 'label_positive_pred'\nsubmission_df = pd.merge(\n    sample_submission_df,\n    predictions_df,\n    on=['ID', 'dataset'],\n    how='left',\n)\n\n# Rename 'label_positive_pred' to 'label_positive' for the final submission format\nsubmission_df = submission_df.rename(columns={'label_positive_pred': 'label_positive_probability'})\nsubmission_df = submission_df.fillna(0.5)\n# Remove the duplicates from the final result, keeping the first occurrence\nsubmission_df = submission_df.drop_duplicates(subset=['ID', 'dataset'], keep='first')\n\nprint(f\"Sample submissions has {len(submission_df)} rows\")\n# Display the head of the final submission DataFrame\nprint(\"Final Submission DataFrame Head:\")\ndisplay(submission_df.head())\n\n# You can save this to a CSV file if needed\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}