{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":106680,"databundleVersionId":13374319,"sourceType":"competition"},{"sourceId":13929627,"sourceType":"datasetVersion","datasetId":8876672},{"sourceId":13931683,"sourceType":"datasetVersion","datasetId":8878189},{"sourceId":13931723,"sourceType":"datasetVersion","datasetId":8878217}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# from glob import glob\n# from collections import Counter\n# import math\n# from tqdm.notebook import tqdm\n# import warnings\n# from sklearn.model_selection import StratifiedKFold\n# from lightgbm import LGBMClassifier\n# from sklearn.metrics import roc_auc_score\n# import os\n# import joblib # <-- NOUVEL IMPORT pour la sauvegarde du modèle\n\n# # --- 1. CONFIGURATION GLOBALE ---\n\n# # Supprimer les warnings pour plus de clarté dans le notebook\n# warnings.filterwarnings('ignore')\n\n# ROOT_DIR = \"/kaggle/input/adaptive-immune-profiling-challenge-2025\"\n# TRAIN_DATA_FOLDER = f\"{ROOT_DIR}/train_datasets/train_datasets\" \n# # Chemin vers le fichier consolidé (Assurez-vous qu'il correspond à votre environnement)\n# TRAIN_METADATA_PATH = '/kaggle/input/airr-all-train-metadata-consolidation/all_train_metadata_consolidated.csv' \n# TARGET_COL = 'label_positive'\n# K_MER_SIZE = 3 \n# N_SPLITS_CV = 5\n# MODEL_FILENAME = 'lgbm_final_model.joblib' # <-- NOUVELLE CONSTANTE\n# ATTRIBUTION_FILENAME = 'df_attributions.csv' # Pour sauvegarder les attributions\n# K_MER_SIZE_ATTR = 3 # Taille du K-mer pour l'attribution\n\n# # Constantes d'encodage\n# AMINO_ACIDS = 'ACDEFGHIKLMNPQRSTVWY'\n# HYDROPHOBICITY = {'A': 1.8, 'C': 2.5, 'D': -3.5, 'E': -3.5, 'F': 2.8, 'G': -0.4, 'H': -3.2, 'I': 4.5, 'K': -3.9, 'L': 3.8, 'M': 1.9, 'N': -3.5, 'P': -1.6, 'Q': -3.5, 'R': -4.5, 'S': -0.8, 'T': -0.7, 'V': 4.2, 'W': -0.9, 'Y': -1.3}\n# TOP_N_KMER = 100 \n# COMMON_V_GENES = [f'TRBV{i}' for i in range(1, 30)] \n# COMMON_J_GENES = [f'TRBJ{i}' for i in range(1, 7)] \n\n# print(\"Configuration chargée.\")\n# print(\"-\" * 50)\n\n\n# # --- 2. DÉFINITIONS DES FONCTIONS (VERSION AMÉLIORÉE) ---\n\n# def calculate_shannon_diversity(template_counts):\n#     \"\"\"Calcule l'entropie de Shannon pour un ensemble de fréquences de clones.\"\"\"\n#     if not template_counts or sum(template_counts) == 0:\n#         return 0.0\n#     total = sum(template_counts)\n#     normalized_probs = [c / total for c in template_counts if c > 0]\n#     shannon_entropy = -sum(p * math.log(p, 2) for p in normalized_probs)\n#     return shannon_entropy\n\n\n# def encode_repertoire_to_features_v2(metadata_df, data_path=None, k_mer_size=K_MER_SIZE, common_v=COMMON_V_GENES, common_j=COMMON_J_GENES, top_kmer=TOP_N_KMER):\n#     \"\"\"\n#     Lit les TSV et extrait des features avancées (v2 + améliorations).\n#     \"\"\"\n#     all_features = []\n    \n#     # Utilisation de tqdm pour la progression (si disponible)\n#     import sys\n#     if 'tqdm.notebook' in sys.modules:\n#         iterator = tqdm(metadata_df.iterrows(), total=len(metadata_df), desc=\"Encodage V2 Amélioré\")\n#     else:\n#         iterator = metadata_df.iterrows()\n    \n#     for index, row in iterator:\n#         tsv_file = row['filename']\n        \n#         try:\n#             rep_df = pd.read_csv(tsv_file, sep='\\t')\n#         except FileNotFoundError:\n#             continue\n\n#         features = row.to_dict()\n        \n#         # --- FEATURE SET 1: Composition & Longueur ---\n        \n#         rep_df['len'] = rep_df['junction_aa'].astype(str).str.len()\n#         features['mean_cdr3_len'] = rep_df['len'].mean()\n#         features['std_cdr3_len'] = rep_df['len'].std()\n        \n#         # AJOUTS : Moments statistiques sur la longueur\n#         features['skew_cdr3_len'] = rep_df['len'].skew()\n#         features['kurtosis_cdr3_len'] = rep_df['len'].kurt()\n        \n#         def get_hydrophobicity(seq):\n#             return sum(HYDROPHOBICITY.get(aa, 0) for aa in seq) / len(seq) if len(seq) > 0 else 0\n            \n#         rep_df['hydrophobicity'] = rep_df['junction_aa'].apply(get_hydrophobicity)\n#         features['mean_hydrophobicity'] = rep_df['hydrophobicity'].mean()\n        \n#         # AJOUT : Écart-type de l'hydrophobie\n#         features['std_hydrophobicity'] = rep_df['hydrophobicity'].std()\n        \n#         # AJOUT : Proportion d'AA invalides/non-fonctionnels\n#         invalid_aa_count = rep_df['junction_aa'].str.contains(r'[^ACDEFGHIKLMNPQRSTVWY]').sum()\n#         features['prop_invalid_aa'] = invalid_aa_count / len(rep_df) if len(rep_df) > 0 else 0\n\n\n#         # --- FEATURE SET 2: Diversité & Structure ---\n#         template_counts = rep_df['templates'].tolist() if 'templates' in rep_df.columns else [1] * len(rep_df)\n#         features['shannon_diversity'] = calculate_shannon_diversity(template_counts)\n#         features['unique_sequences_count'] = len(rep_df)\n#         features['total_templates_sum'] = sum(template_counts)\n#         if 'templates' in rep_df.columns:\n#             sorted_templates = sorted(template_counts, reverse=True)\n#             total_sum = sum(sorted_templates)\n#             features['freq_top_10_clones'] = sum(sorted_templates[:10]) / total_sum if total_sum > 0 else 0\n            \n#         # --- FEATURE SET 3: Gènes V/J Spécifiques & k-mers ---\n#         k_mers = []\n#         for seq in rep_df['junction_aa'].astype(str):\n#             if len(seq) >= k_mer_size:\n#                 k_mers.extend([seq[i:i + k_mer_size] for i in range(len(seq) - k_mer_size + 1)])\n        \n#         kmer_counts = Counter(k_mers)\n#         total_k_mers = sum(kmer_counts.values())\n        \n#         for kmer, count in kmer_counts.most_common(top_kmer):\n#             if all(aa in AMINO_ACIDS for aa in kmer):\n#                 features[f'kmer_freq_{kmer}'] = count / total_k_mers\n\n#         v_counts = rep_df['v_call'].value_counts(normalize=True).to_dict()\n#         j_counts = rep_df['j_call'].value_counts(normalize=True).to_dict()\n        \n#         for gene in common_v:\n#             features[f'v_freq_{gene}'] = v_counts.get(gene, 0)\n#         for gene in common_j:\n#             features[f'j_freq_{gene}'] = j_counts.get(gene, 0)\n            \n#         all_features.append(features)\n\n#     # Création du DataFrame final et nettoyage\n#     df_features = pd.DataFrame(all_features)\n    \n#     # Remplacer les NaNs pour les features numériques *agrégées* si elles manquent\n#     tsvs_cols = ['mean_cdr3_len', 'std_cdr3_len', 'skew_cdr3_len', 'kurtosis_cdr3_len', 'mean_hydrophobicity', 'std_hydrophobicity', 'prop_invalid_aa', 'shannon_diversity', 'unique_sequences_count', 'total_templates_sum', 'freq_top_10_clones']\n#     v_j_kmer_cols = [col for col in df_features.columns if col.startswith(('v_freq_', 'j_freq_', 'kmer_freq_'))]\n    \n#     for col in tsvs_cols + v_j_kmer_cols:\n#         if col in df_features.columns:\n#             df_features[col] = df_features[col].fillna(0)\n    \n#     # Remplacement des NaNs dans le reste du DataFrame (métadonnées) par 0\n#     if TARGET_COL in df_features.columns:\n#         cols_to_fill = df_features.columns.difference([TARGET_COL])\n#         df_features[cols_to_fill] = df_features[cols_to_fill].fillna(0) \n#     else:\n#         df_features = df_features.fillna(0)\n\n#     df_features.columns = df_features.columns.str.replace('[^A-Za-z0-9_]+', '', regex=True)\n    \n#     return df_features\n\n# print(\"✅ Fonctions d'encodage (V2 Amélioré) définies.\")\n# print(\"-\" * 50)\n\n\n# # --- 3. CHARGEMENT & ENCODAGE GLOBAL ---\n\n# print(\"3. Chargement et préparation des données...\")\n\n# # 3.1 Chargement du fichier consolidé\n# all_meta = pd.read_csv(\n#     TRAIN_METADATA_PATH, \n#     sep=',', \n#     skipinitialspace=True\n# )\n\n# # 3.2 Reconstruction du chemin du fichier TSV original\n# all_meta['filename'] = all_meta.apply(\n#     lambda row: os.path.join(ROOT_DIR, 'train_datasets', 'train_datasets', row['dataset'], row['repertoire_id'] + '.tsv'),\n#     axis=1\n# )\n\n# # 3.3 Lancement de l'encodage lourd\n# print(f\"Lancement de l'encodage de {len(all_meta)} répertoires (peut prendre du temps)...\")\n# full_train_df = encode_repertoire_to_features_v2(metadata_df=all_meta)\n\n# print(f\"\\n✅ Encodage global terminé. Ensemble de données final: {full_train_df.shape}\")\n# print(\"-\" * 50)\n\n\n# # --- 4. PRÉPARATION POUR LE ML ---\n\n# # Préparation de X et y\n# attribution_source_df = full_train_df.copy() # Garder une copie pour l'étape 6 (Attribution)\n# X_cols_to_drop = ['repertoire_id', 'filename', TARGET_COL, 'dataset'] \n# X = full_train_df.drop(columns=X_cols_to_drop, errors='ignore')\n# y = full_train_df[TARGET_COL].astype(int)\n\n# # Conversion des types Catégoriels pour LightGBM\n# HLA_COLS = ['A', 'B', 'C', 'DPA1', 'DPB1', 'DQA1', 'DQB1', 'DRB1', 'DRB3', 'DRB4', 'DRB5']\n# CATEGORICAL_COLS = [\n#     'study_group_description', \n#     'sex', \n#     'race', \n#     'sequencing_run_id', \n#     'dataset'\n# ] + HLA_COLS\n\n# for col in CATEGORICAL_COLS:\n#     if col in X.columns:\n#         X[col] = X[col].fillna('Missing').astype('category')\n        \n# # Stocker les catégories pour le jeu de test\n# TRAIN_CATEGORIES = {col: X[col].cat.categories for col in X.columns if X[col].dtype.name == 'category'} \n# X_train_cols = X.columns # Stocker l'ordre des colonnes\n        \n# print(f\"✅ Préparation terminée. X shape: {X.shape}\")\n# print(\"-\" * 50)\n\n\n# # --- 5. MODÉLISATION ET VALIDATION CROISÉE ---\n\n# skf = StratifiedKFold(n_splits=N_SPLITS_CV, shuffle=True, random_state=42)\n# lgbm = LGBMClassifier(\n#     objective='binary', metric='auc', random_state=42, n_estimators=500, n_jobs=-1, verbose=-1,\n#     categorical_feature=[col for col in CATEGORICAL_COLS if col in X.columns] \n# )\n\n# cv_auc_scores = []\n# oof_predictions = np.zeros(len(y))\n\n# print(f\"Début de la Cross-Validation Stratifiée (K={N_SPLITS_CV})...\")\n\n# for fold, (train_index, val_index) in enumerate(skf.split(X, y)):\n#     X_train_fold, X_val_fold = X.iloc[train_index], X.iloc[val_index]\n#     y_train_fold, y_val_fold = y.iloc[train_index], y.iloc[val_index]\n    \n#     lgbm.fit(X_train_fold, y_train_fold)\n#     val_preds = lgbm.predict_proba(X_val_fold)[:, 1]\n#     oof_predictions[val_index] = val_preds\n    \n#     fold_auc = roc_auc_score(y_val_fold, val_preds)\n#     cv_auc_scores.append(fold_auc)\n#     print(f\"  Fold {fold+1}/{N_SPLITS_CV} - AUC: {fold_auc:.4f}\")\n\n# mean_cv_auc = np.mean(cv_auc_scores)\n# oof_auc = roc_auc_score(y, oof_predictions)\n\n# # Rappel du minimum requis (information utilisateur)\n# min_score_required = 0.5 \n\n# print(f\"\\n✨ AUC MOYENNE de la Cross-Validation : {mean_cv_auc:.4f}\")\n# print(f\"⭐ AUC OOF GLOBAL (Estimation Leaderboard) : {oof_auc:.4f}\")\n# print(f\"> Le minimum requis pour la compétition est de {min_score_required}\")\n# print(\"-\" * 50)\n\n\n# # --- 6. ENTRAÎNEMENT DU MODÈLE FINAL, SAUVEGARDE ET ATTRIBUTIONS (OPTIMISÉE) ---\n\n# print(\"6. Entraînement du modèle final, sauvegarde et génération des attributions...\")\n# lgbm.fit(X, y) \n\n# # SAUVEGARDE DU MODÈLE FINAL\n# joblib.dump(lgbm, MODEL_FILENAME)\n# print(f\"✅ Modèle LightGBM FINAL sauvegardé sous : {MODEL_FILENAME}\")\n\n# # -----------------------------------------------------------------------\n# # NOUVELLE SECTION OPTIMISÉE POUR L'ATTRIBUTION (Moins de RAM/Timeout)\n# # -----------------------------------------------------------------------\n\n# print(\"Début de l'extraction des Attributions (Tâche 2)...\")\n\n# # Importance des features liées aux séquences (k-mers, V/J)\n# feature_importances = pd.Series(lgbm.feature_importances_, index=X.columns)\n# relevant_importances = feature_importances[\n#     feature_importances.index.str.startswith(('kmer_freq_', 'v_freq_', 'j_freq_'))]\n\n# # Convertir les importances pertinentes en dictionnaire pour une recherche O(1)\n# importance_dict = relevant_importances.to_dict()\n\n# all_attributions = []\n# MAX_ATTRIBUTIONS_PER_REP = 5000 # Prendre le top 5000 par répertoire\n# MAX_ATTRIBUTIONS_PER_DATASET = 50000 # Limite pour le top N par dataset avant fusion\n\n# # Utiliser attribution_source_df pour lister les répertoires positifs\n# attribution_datasets = attribution_source_df['dataset'].unique()\n\n# for dataset_id in tqdm(attribution_datasets, desc=\"Extraction des Attributions par Dataset\"):\n#     positive_reps = attribution_source_df[(attribution_source_df['dataset'] == dataset_id) & (attribution_source_df[TARGET_COL] == True)]\n    \n#     current_dataset_attributions = []\n    \n#     for _, row in positive_reps.iterrows():\n#         tsv_file = row['filename']\n#         try:\n#             # 1. Lecture du TSV\n#             rep_df = pd.read_csv(tsv_file, sep='\\t')\n#         except FileNotFoundError:\n#             continue\n        \n#         if rep_df.empty:\n#             continue\n        \n#         # Fonction de scoring optimisée qui utilise les dictionnaires d'importance\n#         def calculate_score_fast(junction_aa, v_call, j_call):\n#             score = 0.0\n            \n#             # Score des gènes V et J\n#             score += importance_dict.get(f'v_freq_{v_call}', 0)\n#             score += importance_dict.get(f'j_freq_{j_call}', 0)\n            \n#             # Score des k-mers\n#             if len(junction_aa) >= K_MER_SIZE_ATTR:\n#                 for j in range(len(junction_aa) - K_MER_SIZE_ATTR + 1):\n#                     kmer = junction_aa[j:j + K_MER_SIZE_ATTR]\n#                     score += importance_dict.get(f'kmer_freq_{kmer}', 0)\n            \n#             return score\n\n#         # 3. Application du score sur les colonnes\n#         rep_df['importance_score'] = rep_df.apply(\n#             lambda x: calculate_score_fast(x['junction_aa'], x['v_call'], x['j_call']), \n#             axis=1\n#         )\n        \n#         # 4. Collection des Top Séquences (Limitation de la RAM)\n#         rep_df_sorted = rep_df.sort_values(by='importance_score', ascending=False)\n#         # Prendre le top N par répertoire pour ne pas surcharger la mémoire\n#         top_sequence_info = rep_df_sorted[['junction_aa', 'v_call', 'j_call', 'importance_score']].head(MAX_ATTRIBUTIONS_PER_REP) \n#         top_sequence_info['dataset'] = dataset_id\n        \n#         current_dataset_attributions.append(top_sequence_info)\n\n#     # Consolidation des attributions pour le dataset en cours\n#     if current_dataset_attributions:\n#         combined_attributions = pd.concat(current_dataset_attributions)\n        \n#         # Filtrer et conserver le top 50 000 par dataset, basé sur le score global\n#         final_attributions_for_dataset = combined_attributions.sort_values(\n#             by='importance_score', ascending=False\n#         ).head(MAX_ATTRIBUTIONS_PER_DATASET)\n        \n#         all_attributions.append(final_attributions_for_dataset)\n\n# # -----------------------------------------------------------------------\n# # FINALISATION DE L'ARTEFACT D'ATTRIBUTION\n# # -----------------------------------------------------------------------\n\n# if all_attributions:\n#     df_attributions = pd.concat(all_attributions).reset_index(drop=True)\n    \n#     # Nous gardons le TOP 400k des attributions selon les règles courantes (à ajuster si besoin)\n#     MAX_TOTAL_ATTRIBUTIONS = 400000 \n#     df_attributions = df_attributions.sort_values(by='importance_score', ascending=False).head(MAX_TOTAL_ATTRIBUTIONS)\n\n#     # Préparation finale pour la soumission\n#     df_attributions = df_attributions[['dataset', 'junction_aa', 'v_call', 'j_call']]\n    \n#     # Sauvegarde finale\n#     ATTRIBUTION_FILENAME = 'df_attributions.csv'\n#     df_attributions.to_csv(ATTRIBUTION_FILENAME, index=False)\n    \n#     print(f\"✅ Attributions finalisées. Total de lignes : {len(df_attributions)}\")\n#     print(f\"✅ Attributions sauvegardées sous : {ATTRIBUTION_FILENAME}\")\n\n# else:\n#     print(\"❌ Avertissement : Aucune attribution générée.\")\n# print(\"-\" * 50)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------------------------------------\n# CELLE COMPLÈTE POUR SOUMISSION TEST + ATTRIBUTIONS SÉCURISÉE\n# (FINAL FIX: Remplacement du slicing dangereux par une reconstruction sûre pour les IDs dupliqués)\n# -------------------------------------------------------------\nimport pandas as pd\nimport numpy as np\nimport joblib\nfrom glob import glob\nfrom collections import Counter\nfrom itertools import cycle\nimport math\nimport os\nfrom tqdm.notebook import tqdm\nfrom typing import List\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\n# ------------------- CONSTANTES -------------------\nROOT_DIR = \"/kaggle/input/adaptive-immune-profiling-challenge-2025\"\nTEST_DATA_FOLDER = f\"{ROOT_DIR}/test_datasets/test_datasets\"\nSUBMISSION_BASE_PATH = f\"{ROOT_DIR}/sample_submissions.csv\"\nMETADATA_STUDIES_PATH = f\"{ROOT_DIR}/metadata_studies.csv\"\nATTRIBUTION_FILENAME = '/kaggle/input/airr-df-attributions/df_attributions.csv'\nMODEL_PATH = '/kaggle/input/lgbm-final-model/lgbm_final_model.joblib'\n\nTARGET_COL = 'label_positive'\nK_MER_SIZE = 3\nAMINO_ACIDS = 'ACDEFGHIKLMNPQRSTVWY'\nCOMMON_V_GENES = [f'TRBV{i}' for i in range(1, 30)]\nCOMMON_J_GENES = [f'TRBJ{i}' for i in range(1, 7)]\nMISSING_VALUE_REPLACEMENT = 0.0 # Remplacement des NaNs dans les features numériques (Train/Test)\nMISSING_SUBMISSION_VALUE = -999.0 # Valeur requise pour la section d'Attribution\nSUBMISSION_FILENAME = 'submission.csv'\n\nCATEGORICAL_COLS = [\n    'study_group_description', 'sex', 'race', 'sequencing_run_id', 'dataset',\n    'A', 'B', 'C', 'DPA1', 'DPB1', 'DQA1', 'DQB1', 'DRB1', 'DRB3', 'DRB4', 'DRB5'\n]\n\nHYDROPHOBICITY = {'A': 1.8, 'C': 2.5, 'D': -3.5, 'E': -3.5, 'F': 2.8, 'G': -0.4,\n                  'H': -3.2, 'I': 4.5, 'K': -3.9, 'L': 3.8, 'M': 1.9, 'N': -3.5,\n                  'P': -1.6, 'Q': -3.5, 'R': -4.5, 'S': -0.8, 'T': -0.7, 'V': 4.2,\n                  'W': -0.9, 'Y': -1.3}\n\n# ------------------- MOCK/SETUP POUR ROBUSTESSE -------------------\nclass MockModel:\n    def __init__(self, feature_name): self.feature_name_ = feature_name\n    def predict_proba(self, X): return np.array([[0.5, 0.5]] * len(X))\n\n# ------------------- FONCTIONS UTILITAIRES -------------------\ndef calculate_shannon_diversity(template_counts):\n    if not template_counts or sum(template_counts) == 0:\n        return 0.0\n    total = sum(template_counts)\n    probs = [c / total for c in template_counts if c > 0]\n    return -sum(p * math.log(p, 2) for p in probs)\n\ndef encode_repertoire_to_features_v2(metadata_df: pd.DataFrame, x_train_cols_ref: List[str], k_mer_size=K_MER_SIZE, common_v=COMMON_V_GENES, common_j=COMMON_J_GENES):\n    all_features = []\n    # TRAIN_KMER_COLS contient tous les K-mers que le modèle connaît\n    TRAIN_KMER_COLS = [col.replace('kmer_freq_', '') for col in x_train_cols_ref if col.startswith('kmer_freq_')]\n    \n    for _, row in tqdm(metadata_df.iterrows(), total=len(metadata_df), desc=\"Encodage Test V2\"):\n        tsv_file = row['filename']\n        features = row.to_dict()\n        \n        try:\n            rep_df = pd.read_csv(tsv_file, sep='\\t', engine='c')\n        except Exception:\n            all_features.append(features)\n            continue\n        if 'junction_aa' not in rep_df.columns:\n            all_features.append(features)\n            continue\n\n        # --- Features longueur & hydrophobicité ---\n        rep_df['len'] = rep_df['junction_aa'].astype(str).str.len()\n        features['mean_cdr3_len'] = rep_df['len'].mean()\n        features['std_cdr3_len'] = rep_df['len'].std()\n        features['skew_cdr3_len'] = rep_df['len'].skew()\n        features['kurtosis_cdr3_len'] = rep_df['len'].kurt()\n        rep_df['hydrophobicity'] = rep_df['junction_aa'].apply(lambda seq: sum(HYDROPHOBICITY.get(aa, 0) for aa in seq)/len(seq) if len(seq)>0 else 0)\n        features['mean_hydrophobicity'] = rep_df['hydrophobicity'].mean()\n        features['std_hydrophobicity'] = rep_df['hydrophobicity'].std()\n        invalid_aa_count = rep_df['junction_aa'].str.contains(r'[^ACDEFGHIKLMNPQRSTVWY]').sum()\n        features['prop_invalid_aa'] = invalid_aa_count / len(rep_df) if len(rep_df) > 0 else 0\n\n        # --- Diversité & clonotypes ---\n        template_counts = rep_df['templates'].tolist() if 'templates' in rep_df.columns else [1]*len(rep_df)\n        features['shannon_diversity'] = calculate_shannon_diversity(template_counts)\n        features['unique_sequences_count'] = len(rep_df)\n        features['total_templates_sum'] = sum(template_counts)\n        total_sum_templates = sum(template_counts)\n        sorted_templates = sorted(template_counts, reverse=True)\n        features['freq_top_10_clones'] = sum(sorted_templates[:10]) / total_sum_templates if total_sum_templates > 0 else 0\n\n        # --- k-mers (alignement FORCÉ au vocabulaire train) ---\n        k_mers = []\n        for seq in rep_df['junction_aa'].dropna().astype(str):\n            if len(seq) >= k_mer_size:\n                k_mers.extend([seq[i:i+k_mer_size] for i in range(len(seq)-k_mer_size+1)])\n        kmer_counts = Counter(k_mers)\n        total_k_mers = sum(kmer_counts.values())\n\n        # Créer toutes les colonnes K-mers de Train, même si non trouvées (valeur 0.0)\n        for kmer_name in TRAIN_KMER_COLS:\n            count = kmer_counts.get(kmer_name, 0)\n            features[f'kmer_freq_{kmer_name}'] = count / total_k_mers if total_k_mers > 0 else 0.0\n\n        # --- V/J gene frequencies ---\n        v_counts = rep_df['v_call'].fillna('').value_counts(normalize=True).to_dict() if 'v_call' in rep_df.columns else {}\n        j_counts = rep_df['j_call'].fillna('').value_counts(normalize=True).to_dict() if 'j_call' in rep_df.columns else {}\n        for gene in common_v: features[f'v_freq_{gene}'] = v_counts.get(gene, 0)\n        for gene in common_j: features[f'j_freq_{gene}'] = j_counts.get(gene, 0)\n\n        all_features.append(features)\n\n    df_features = pd.DataFrame(all_features)\n    df_features.columns = df_features.columns.str.replace('[^A-Za-z0-9_]+', '', regex=True)\n    return df_features\n\n# ------------------- CHARGEMENT ET PRÉPARATION -------------------\nprint(\"1. Chargement des artefacts de base...\")\n\n# 1) Load model (and get X_train_cols)\ntry:\n    lgbm = joblib.load(MODEL_PATH)\n    print(\"✅ Modèle LGBM chargé.\")\n    \n    try:\n        if hasattr(lgbm, \"feature_name_\"):\n            X_train_cols = list(lgbm.feature_name_)\n        elif hasattr(lgbm, \"base_estimator_\") and hasattr(lgbm.base_estimator_, \"feature_name_\"):\n            X_train_cols = list(lgbm.base_estimator_.feature_name_)\n        else:\n            raise AttributeError(\"Attribut feature_name_ introuvable sur le modèle chargé.\")\n        print(f\"✅ {len(X_train_cols)} features d'entraînement trouvées.\")\n        \n    except AttributeError as ae:\n        print(f\"⚠️ Erreur lors de la récupération des features : {ae}. Utilisation du MockModel pour les colonnes.\")\n        # Utilisation du MockModel si les colonnes ne peuvent pas être récupérées\n        mock_kmers = [f'kmer_freq_{aa1}{aa2}{aa3}' for aa1 in AMINO_ACIDS for aa2 in AMINO_ACIDS for aa3 in AMINO_ACIDS][:100]\n        X_train_cols = ['mean_cdr3_len', 'shannon_diversity'] + mock_kmers + CATEGORICAL_COLS\n        lgbm = MockModel(X_train_cols)\n        print(\"⚠️ ALERTE : Soumission finale doit utiliser le VRAI modèle, pas le MockModel (probabilité 0.5).\")\n\nexcept Exception as e:\n    print(f\"❌ Impossible de charger le modèle LGBM depuis {MODEL_PATH} ({e}). Utilisation d'un MockModel.\")\n    # Utilisation du MockModel si le modèle ne peut pas être chargé\n    mock_kmers = [f'kmer_freq_{aa1}{aa2}{aa3}' for aa1 in AMINO_ACIDS for aa2 in AMINO_ACIDS for aa3 in AMINO_ACIDS][:100]\n    X_train_cols = ['mean_cdr3_len', 'shannon_diversity'] + mock_kmers + CATEGORICAL_COLS\n    lgbm = MockModel(X_train_cols)\n    print(\"⚠️ ALERTE : Soumission finale doit utiliser le VRAI modèle, pas le MockModel (probabilité 0.5).\")\n\n\n# 2) Load submission base (we want to work with repertoire_id internally)\ntry:\n    # Lecture du fichier de soumission de base\n    # NOTE IMPORTANTE: Nous renommons ID_original pour le nettoyer plus tard et éviter la duplication ID/ID\n    df_submission_base = pd.read_csv(SUBMISSION_BASE_PATH).rename(columns={'ID':'ID_original', 'repertoire_id':'ID_original'})\n    \n    # S'assurer que le nom repertoire_id est correct pour la fusion\n    if 'ID_original' in df_submission_base.columns:\n        # Renommer la colonne ID_original en 'repertoire_id' pour la logique de prédiction\n        df_submission_base.rename(columns={'ID_original':'repertoire_id'}, inplace=True)\n        # Mais conserver l'autre 'ID' si elle existe pour la soumission finale (bien que vide)\n        if len(df_submission_base.filter(like='ID_original').columns) > 1:\n            df_submission_base.drop(columns=df_submission_base.filter(like='ID_original').columns[1:], inplace=True)\n\n\n    # FIX: Supprimer toute colonne d'index non nommée (qui crée le ID dupliqué)\n    cols_to_drop_on_load = [col for col in df_submission_base.columns if 'Unnamed:' in str(col)]\n    df_submission_base.drop(columns=cols_to_drop_on_load, inplace=True, errors='ignore')\n    \n    # S'assurer que l'ID principal est bien 'repertoire_id'\n    if 'ID' in df_submission_base.columns and 'repertoire_id' not in df_submission_base.columns:\n        df_submission_base.rename(columns={'ID':'repertoire_id'}, inplace=True)\n    elif 'ID' in df_submission_base.columns and 'repertoire_id' in df_submission_base.columns:\n        # Si les deux existent, garder repertoire_id et supprimer ID (le vide/dupliqué)\n        df_submission_base.drop(columns=['ID'], inplace=True, errors='ignore')\n\n    # Reconfirmer l'ID à prédire\n    repertoire_ids_to_predict = df_submission_base[df_submission_base['repertoire_id'].astype(str)!=str(MISSING_SUBMISSION_VALUE)]['repertoire_id'].tolist()\n    print(f\"✅ Fichier de soumission de base chargé. {len(df_submission_base)} lignes totales.\")\nexcept FileNotFoundError:\n    # MOCK DATA\n    df_submission_base = pd.DataFrame({\n        'repertoire_id':[f'mock_{i}' for i in range(100)],\n        'dataset':['dataset_1']*100,\n        'label_positive_probability':[0.5]*100,\n        'junction_aa':['']*100,\n        'v_call':['']*100,\n        'j_call':['']*100\n    })\n    df_submission_base.loc[50:, 'repertoire_id'] = MISSING_SUBMISSION_VALUE\n    repertoire_ids_to_predict = df_submission_base[df_submission_base['repertoire_id'].astype(str)!=str(MISSING_SUBMISSION_VALUE)]['repertoire_id'].tolist()\n    print(f\"⚠️ sample_submissions non trouvé -> mock créé. {len(repertoire_ids_to_predict)} répertoires à prédire.\")\n\n# Métadonnées et Attributions\ntry:\n    df_studies = pd.read_csv(METADATA_STUDIES_PATH).rename(columns={'ID':'repertoire_id'})\n    df_studies['repertoire_id'] = df_studies['repertoire_id'].astype(str)\nexcept FileNotFoundError:\n    df_studies = pd.DataFrame(columns=['repertoire_id'] + CATEGORICAL_COLS)\ntry:\n    df_attributions = pd.read_csv(ATTRIBUTION_FILENAME).fillna(\"\")\nexcept FileNotFoundError:\n    df_attributions = pd.DataFrame(columns=['junction_aa','v_call','j_call','dataset'])\n\n# Mapping des fichiers TSV (test)\nall_test_tsv_paths = glob(f\"{TEST_DATA_FOLDER}/**/*.tsv\", recursive=True)\ntest_tsv_path_mapping = {}\nfor path in all_test_tsv_paths:\n    repertoire_id = os.path.splitext(os.path.basename(path))[0]\n    dataset_id = os.path.basename(os.path.dirname(path))\n    if repertoire_id in repertoire_ids_to_predict:\n        test_tsv_path_mapping[repertoire_id] = {'repertoire_id':repertoire_id,'filename':path,'dataset':dataset_id,'dataset_id':dataset_id}\n\nall_test_meta = pd.DataFrame.from_dict(test_tsv_path_mapping, orient='index').reset_index(drop=True)\nif not all_test_meta.empty:\n    all_test_meta['filename'] = all_test_meta['filename'].str.replace('\\\\','/',regex=False)\nall_test_meta = all_test_meta.merge(df_studies.drop(columns=['dataset_id','dataset'],errors='ignore'), on='repertoire_id', how='left')\n\nprint(\"2. Encodage des features...\")\nfull_test_df = encode_repertoire_to_features_v2(all_test_meta, x_train_cols_ref=X_train_cols)\nVALID_TEST_DATASETS = full_test_df['dataset'].unique().tolist() if 'dataset' in full_test_df.columns else []\nif not VALID_TEST_DATASETS:\n    VALID_TEST_DATASETS = ['dataset_1']\n\n# --- Logging de débogage pour les K-mers ---\nKMER_COLS_CHECK = [col for col in full_test_df.columns if col.startswith('kmer_freq_')]\nif not KMER_COLS_CHECK:\n    print(\"⚠️ Attention: Aucune colonne de k-mer n'a été trouvée dans les features de test.\")\nelse:\n    # Compter le nombre de colonnes de K-mers où au moins un répertoire a une fréquence > 0\n    non_zero_kmer_count = (full_test_df[KMER_COLS_CHECK] != 0.0).any(axis=0).sum()\n    print(f\"🔍 DEBUG K-MERS: {non_zero_kmer_count} colonnes de k-mers sur {len(KMER_COLS_CHECK)} ont des valeurs non-nulles dans le jeu de test.\")\n# ----------------------------------------------------------------------------------\n\n# ------------------- ALIGNEMENT ET PRÉDICTION -------------------\n# Build X_test (index = repertoire_id)\nif 'repertoire_id' not in full_test_df.columns:\n    full_test_df = full_test_df.reset_index().rename(columns={'index':'repertoire_id'})\n\nX_test = full_test_df.set_index('repertoire_id').drop(columns=['filename','dataset_id'], errors='ignore')\n\n# Restrict to rows that are present in sample_submissions (safety)\npred_ids = list(set(X_test.index).intersection(set(repertoire_ids_to_predict)))\nX_test = X_test.loc[pred_ids]\n\n# Prepare aligned DF with correct columns/order from training\nX_test_aligned = pd.DataFrame(index=X_test.index, columns=X_train_cols)\n\n# Precompute numerical and categorical columns lists\nNUMERICAL_COLS = [col for col in X_train_cols if col not in CATEGORICAL_COLS]\n\n# 🎯 FIX MAJEUR D'ALIGNEMENT: Remplissage et alignement forcé\nfor col in X_train_cols:\n    if col in X_test.columns:\n        X_test_aligned[col] = X_test[col]\n    else:\n        # Si la colonne est connue à l'entraînement mais absente de l'encodage test,\n        # elle sera remplie par NaN ici, puis imputée correctement par la suite.\n        X_test_aligned[col] = np.nan\n\n# Categorical handling: replace NaN with 'Missing' and set dtype 'category'\nfor col in CATEGORICAL_COLS:\n    if col in X_test_aligned.columns:\n        # Si le modèle a été entraîné avec des catégories (LightGBM), on doit les conserver.\n        X_test_aligned[col] = X_test_aligned[col].fillna('Missing').astype(str)\n        X_test_aligned[col] = pd.Categorical(X_test_aligned[col])\n\n# Numeric handling: coerce to numeric then impute\nfor col in NUMERICAL_COLS:\n    if col in X_test_aligned.columns:\n        X_test_aligned[col] = pd.to_numeric(X_test_aligned[col], errors='coerce')\n\n# Imputation de 0.0 UNIQUEMENT sur les colonnes numériques (CORRECTIF)\nX_test_aligned[NUMERICAL_COLS] = X_test_aligned[NUMERICAL_COLS].fillna(MISSING_VALUE_REPLACEMENT)\n\n# Final safety: ensure index sorted same as predictions list\nX_test_aligned = X_test_aligned.loc[sorted(X_test_aligned.index)]\n\n# Predict (handle empty gracefully)\nif len(X_test_aligned) > 0:\n    test_preds_proba = lgbm.predict_proba(X_test_aligned)[:,1]\n    df_predictions = pd.DataFrame({'repertoire_id':X_test_aligned.index,'label_positive_probability':test_preds_proba})\n    print(f\"✅ Prédictions effectuées pour {len(df_predictions)} répertoires.\")\nelse:\n    df_predictions = pd.DataFrame({'repertoire_id':[],'label_positive_probability':[]})\n    print(\"⚠️ Aucun répertoire à prédire (X_test_aligned vide).\")\n\n# ------------------- SOUMISSION -------------------\n# Classification section (predicted rows)\ndf_classification_section = df_submission_base[df_submission_base['repertoire_id'].isin(df_predictions['repertoire_id'])].copy()\nif 'dataset' in full_test_df.columns:\n    df_classification_section['dataset'] = df_classification_section['repertoire_id'].map(full_test_df.set_index('repertoire_id')['dataset']).fillna(df_classification_section.get('dataset', np.nan))\ndf_classification_section = df_classification_section.merge(df_predictions, on='repertoire_id', how='left', suffixes=('_base','_pred'))\ndf_classification_section['label_positive_probability'] = df_classification_section['label_positive_probability_pred'].fillna(MISSING_VALUE_REPLACEMENT)\ndf_classification_section = df_classification_section[['repertoire_id','dataset','label_positive_probability','junction_aa','v_call','j_call']]\n\n\n# 🎯 FIX 2: Attribution section (Robust logic for identifying -999.0 rows)\nclassified_ids = df_classification_section['repertoire_id'].tolist()\n# Get all rows from base that were NOT in the classification section (ce sont les -999.0)\ndf_attribution_section = df_submission_base[~df_submission_base['repertoire_id'].isin(classified_ids)].copy()\n\nrequired_len = len(df_attribution_section)\ncols_to_map = ['junction_aa','v_call','j_call']\n\n# Keep only attributions that belong to VALID_TEST_DATASETS\ndf_attributions = df_attributions[df_attributions['dataset'].isin(VALID_TEST_DATASETS)].copy() if not df_attributions.empty else df_attributions\ndf_attributions_mapped = df_attributions[cols_to_map].head(required_len) if not df_attributions.empty else pd.DataFrame(columns=cols_to_map)\n\n# If not enough attributions, pad with empty strings (Confirmation: c'est la bonne pratique)\nif len(df_attributions_mapped) < required_len:\n    missing_rows = required_len - len(df_attributions_mapped)\n    # Assurer que les colonnes manquantes sont initialisées\n    missing_data = {col: [\"\"]*missing_rows for col in cols_to_map}\n    df_attributions_mapped = pd.concat([df_attributions_mapped, pd.DataFrame(missing_data)], ignore_index=True)\n    df_attributions_mapped = df_attributions_mapped.iloc[:required_len] # Tronquer si trop de lignes ajoutées/existantes\n\nif required_len > 0:\n    df_attribution_section['junction_aa'] = df_attributions_mapped['junction_aa'].values\n    df_attribution_section['v_call'] = df_attributions_mapped['v_call'].values\n    df_attribution_section['j_call'] = df_attributions_mapped['j_call'].values\n    dataset_cycle = cycle(VALID_TEST_DATASETS)\n    df_attribution_section['dataset'] = [next(dataset_cycle) for _ in range(required_len)]\n    # Use -999.0 for attribution probability column as required by the competition\n    df_attribution_section['label_positive_probability'] = float(MISSING_SUBMISSION_VALUE)\n\n# rename attribution section's repertoire_id -> ID (as expected in final file)\ndf_attribution_section.rename(columns={'repertoire_id':'ID'}, inplace=True)\n\n# Concatenate classification + attribution\ndf_final_submission = pd.concat([df_classification_section, df_attribution_section], ignore_index=True)\n# For the classification rows we still have 'repertoire_id' column; rename it to 'ID' to match sample format\nif 'repertoire_id' in df_final_submission.columns:\n    df_final_submission.rename(columns={'repertoire_id':'ID'}, inplace=True)\n\n\n# 🎯 FIX 3: Nettoyage et élimination de la colonne ID dupliquée/vide (avant l'export)\n# Supprimer les colonnes d'index non nécessaires ou les duplicata d'ID accidentels.\ncols_to_drop = [col for col in df_final_submission.columns if 'Unnamed:' in str(col)]\ndf_final_submission.drop(columns=cols_to_drop, inplace=True, errors='ignore')\n\n# 🎯 FIX ULTIME 3.0 (CORRIGÉ V5): Remplacement du Slicing Dangereux par une Reconstruction Sûre\nif list(df_final_submission.columns).count(\"ID\") > 1:\n    cols = df_final_submission.columns\n    id_cols = [i for i, c in enumerate(cols) if c == \"ID\"]\n    keep = id_cols[-1]\n\n    new_cols = []\n    for i, c in enumerate(cols):\n        if c != \"ID\":\n            new_cols.append(c)\n        else:\n            if i == keep:\n                new_cols.append(c)  # garder uniquement cette ID\n\n    # Cette opération de reconstruction est la plus robuste (remplace le slicing ambigu)\n    df_final_submission = df_final_submission[new_cols] \n\n\n# Build expected column order from original sample_submissions but adapt 'repertoire_id' -> 'ID'\noriginal_cols = df_submission_base.columns.tolist()\nexpected_cols = [c if c != 'repertoire_id' else 'ID' for c in original_cols]\n# Rendre la liste des colonnes attendues unique (pour gérer les IDs dupliqués accidentels)\nexpected_cols_unique = []\nseen = set()\nfor col in expected_cols:\n    if col not in seen:\n        expected_cols_unique.append(col)\n        seen.add(col)\nexpected_cols = expected_cols_unique\n    \n# final selection - if some expected columns are missing, keep intersection but warn\nmissing_expected = [c for c in expected_cols if c not in df_final_submission.columns]\nif missing_expected:\n    print(f\"⚠️ Colonnes attendues manquantes dans df_final_submission (elles seront ajoutées en tant que vide) : {missing_expected}\")\n    for c in missing_expected:\n        df_final_submission[c] = \"\" # pad missing columns with empty strings\n\ndf_final_submission = df_final_submission[expected_cols]\n\n# ------------------- SUPER-SAFE TYPE FIX (CORRIGÉ V4) -------------------\n\n# Colonnes obligatoires pour strings\nSTRING_COLUMNS = ['ID', 'dataset', 'junction_aa', 'v_call', 'j_call']\nNUMERIC_COLUMNS = ['label_positive_probability']\n\n# 1. Assurer que toutes les colonnes string sont bien remplies et en str\nfor col in STRING_COLUMNS:\n    if col in df_final_submission.columns:\n        df_final_submission[col] = df_final_submission[col].fillna(\"\").astype(str)\n\n# 2. Colonnes numériques (probabilité)\nfor col in NUMERIC_COLUMNS:\n    if col in df_final_submission.columns:\n        # Coerce to numeric (float), remplace les non-convertibles par MISSING_SUBMISSION_VALUE (-999.0)\n        df_final_submission[col] = pd.to_numeric(df_final_submission[col], errors='coerce').fillna(MISSING_SUBMISSION_VALUE).astype(float)\n\n# 3. Sécurité ultime: s'assurer qu'aucune colonne n'est object inattendue\n# L'objet est maintenant garanti d'être un DataFrame (grâce à FIX 3.0 V5)\nfor col, dtype in df_final_submission.dtypes.items():\n    if dtype == 'object' and col not in STRING_COLUMNS:\n        # Convertir tout objet restant en string (sécurité ultime)\n        df_final_submission[col] = df_final_submission[col].astype(str)\n\nprint(\"✅ Types forcés avec succès avant export.\")\n# ------------------------------------------------------------------------------------------------------------------------------------\n\n# ====================================================================\n# 🔍 BLOC D'INSPECTION DU DATAFRAME FINAL\n# ====================================================================\nprint(\"\\n--- 🔍 INSPECTION DU DATAFRAME FINAL ---\")\nprint(f\"Total des lignes: {len(df_final_submission)}\")\n\n# Afficher les 5 premières lignes (Vérification des IDs et probabilités)\nprint(\"\\n[HEAD - 5 premières lignes]\")\nprint(df_final_submission[['ID', 'label_positive_probability', 'junction_aa']].head())\n\n# Afficher les 5 dernières lignes (Vérification du padding -999.0)\nprint(\"\\n[TAIL - 5 dernières lignes]\")\nprint(df_final_submission[['ID', 'label_positive_probability', 'junction_aa']].tail())\n\n# Vérification des types de colonnes\nprint(\"\\n[DTYPES - Vérification des types de colonnes]\")\nprint(df_final_submission.dtypes)\n\n# Vérification rapide des valeurs spéciales\nnum_minus_999 = (df_final_submission['label_positive_probability'] == float(MISSING_SUBMISSION_VALUE)).sum()\nprint(f\"\\nNombre de lignes d'attribution (-999.0): {num_minus_999}\")\nprint(\"-------------------------------------------\\n\")\n# ====================================================================\n\n\n# ------------------- CHECK SÉCURITÉ -------------------\nif 'dataset' in df_final_submission.columns:\n    train_datasets_in_submission = [d for d in df_final_submission['dataset'].unique() if str(d).startswith('train')]\n    if train_datasets_in_submission:\n        raise ValueError(f\"❌ Train datasets présents dans la soumission : {train_datasets_in_submission}\")\n\n# Export\ndf_final_submission.to_csv(SUBMISSION_FILENAME, index=False)\nprint(f\"🎉 Fichier de soumission final créé: {SUBMISSION_FILENAME}\")\nprint(f\"Nombre total de lignes: {len(df_final_submission)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}