{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install xgboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T12:28:50.704082Z","iopub.execute_input":"2025-06-22T12:28:50.704437Z","iopub.status.idle":"2025-06-22T12:28:55.445226Z","shell.execute_reply.started":"2025-06-22T12:28:50.704401Z","shell.execute_reply":"2025-06-22T12:28:55.439061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===================================================================\n# 🏆 COMPLETE FIXED XGBOOST RANKER - HANDLES MISSING FEATURES 🏆\n# ===================================================================\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport xgboost as xgb\nfrom sklearn.model_selection import GroupKFold, train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import ndcg_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n\nprint(\"🚀 Starting Advanced Flight Ranking Analysis\")\nprint(\"=\"*60)\n\n# ===================================================================\n# 📊 DATA LOADING AND INITIAL EXPLORATION\n# ===================================================================\n\n# Load competition data\ntrain_df = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet')\nsample_submission = pd.read_parquet('/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet')\n\nprint(f\"📈 Dataset Overview:\")\nprint(f\"   Training data shape: {train_df.shape}\")\nprint(f\"   Sample submission shape: {sample_submission.shape}\")\nprint(f\"   Memory usage: {train_df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n\n# Display basic info about the dataset\nprint(f\"\\n🔍 Data Types Distribution:\")\nprint(train_df.dtypes.value_counts())\n\nprint(f\"\\n📋 Column Overview:\")\nprint(f\"   Total columns: {len(train_df.columns)}\")\nprint(f\"   Numeric columns: {len(train_df.select_dtypes(include=[np.number]).columns)}\")\nprint(f\"   Object columns: {len(train_df.select_dtypes(include=['object']).columns)}\")\n\n# ===================================================================\n# 🔧 UTILITY FUNCTIONS FOR DATA TYPE HANDLING\n# ===================================================================\n\ndef fix_boolean_columns(df, column_name):\n    \"\"\"\n    Fix boolean columns that contain string representations\n    FIXED: Handles 'False'/'True' strings properly\n    \"\"\"\n    if column_name not in df.columns:\n        return df\n    \n    # Check if column contains string boolean values\n    if df[column_name].dtype == object:\n        # Handle various string representations of boolean values\n        bool_mapping = {\n            'False': False, 'True': True, \n            'false': False, 'true': True,\n            'FALSE': False, 'TRUE': True,\n            '0': False, '1': True,\n            0: False, 1: True,\n            False: False, True: True\n        }\n        \n        # Apply mapping and fill any remaining NaN with False\n        df[column_name] = df[column_name].map(bool_mapping).fillna(False)\n    \n    # Convert to boolean type first, then to int\n    df[column_name] = df[column_name].astype(bool).astype(int)\n    return df\n\ndef safe_fillna_by_dtype(series, fill_value):\n    \"\"\"\n    Safely fill NaN values based on column data type\n    FIXED: Handles Int64 and other nullable dtypes properly\n    \"\"\"\n    if pd.api.types.is_integer_dtype(series.dtype):\n        # For integer columns (including Int64), use integer fill values\n        if isinstance(fill_value, str):\n            # Convert string to appropriate integer (use mode or median)\n            if series.notna().sum() > 0:\n                fill_value = int(series.mode().iloc[0]) if len(series.mode()) > 0 else 0\n            else:\n                fill_value = 0\n        return series.fillna(fill_value)\n    \n    elif pd.api.types.is_float_dtype(series.dtype):\n        # For float columns\n        if isinstance(fill_value, str):\n            fill_value = series.median() if series.notna().sum() > 0 else 0.0\n        return series.fillna(fill_value)\n    \n    elif pd.api.types.is_bool_dtype(series.dtype):\n        # For boolean columns\n        if isinstance(fill_value, str):\n            fill_value = False\n        return series.fillna(fill_value)\n    \n    else:\n        # For object/string columns\n        return series.fillna(str(fill_value))\n\n# ===================================================================\n# 🔍 DEEP EXPLORATORY DATA ANALYSIS\n# ===================================================================\n\ndef comprehensive_eda(df):\n    \"\"\"\n    Perform comprehensive exploratory data analysis\n    This is crucial for understanding the ranking problem structure\n    \"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(\"🔬 COMPREHENSIVE EXPLORATORY DATA ANALYSIS\")\n    print(\"=\"*60)\n    \n    # Target variable analysis\n    print(f\"\\n🎯 Target Variable Analysis:\")\n    target_dist = df['selected'].value_counts().sort_index()\n    print(target_dist)\n    \n    # Plot target distribution\n    fig, axes = plt.subplots(2, 2, figsize=(15, 12))\n    \n    # Target distribution\n    axes[0,0].bar(target_dist.index, target_dist.values, color='skyblue', alpha=0.7)\n    axes[0,0].set_title('Target Distribution (Selected Flights)', fontsize=14, fontweight='bold')\n    axes[0,0].set_xlabel('Selected (0=No, 1=Yes)')\n    axes[0,0].set_ylabel('Count')\n    \n    # Ranker ID analysis\n    ranker_stats = df.groupby('ranker_id').agg({\n        'selected': ['count', 'sum', 'mean']\n    }).round(3)\n    ranker_stats.columns = ['Total_Flights', 'Selected_Flights', 'Selection_Rate']\n    \n    print(f\"\\n🔢 Ranker ID Statistics:\")\n    print(f\"   Unique ranker_ids: {df['ranker_id'].nunique()}\")\n    print(f\"   Avg flights per ranker: {ranker_stats['Total_Flights'].mean():.2f}\")\n    print(f\"   Avg selection rate: {ranker_stats['Selection_Rate'].mean():.3f}\")\n    \n    # Plot ranker statistics\n    axes[0,1].hist(ranker_stats['Total_Flights'], bins=30, color='lightcoral', alpha=0.7)\n    axes[0,1].set_title('Distribution of Flights per Ranker', fontsize=14, fontweight='bold')\n    axes[0,1].set_xlabel('Number of Flights')\n    axes[0,1].set_ylabel('Frequency')\n    \n    # Price analysis\n    print(f\"\\n💰 Price Analysis:\")\n    print(f\"   Total Price - Mean: ${df['totalPrice'].mean():.2f}, Std: ${df['totalPrice'].std():.2f}\")\n    print(f\"   Taxes - Mean: ${df['taxes'].mean():.2f}, Std: ${df['taxes'].std():.2f}\")\n    print(f\"   Tax Ratio - Mean: {(df['taxes']/df['totalPrice']).mean():.3f}\")\n    \n    # Price distributions\n    axes[1,0].hist(df['totalPrice'], bins=50, color='lightgreen', alpha=0.7)\n    axes[1,0].set_title('Total Price Distribution', fontsize=14, fontweight='bold')\n    axes[1,0].set_xlabel('Total Price ($)')\n    axes[1,0].set_ylabel('Frequency')\n    \n    axes[1,1].scatter(df['totalPrice'], df['taxes'], alpha=0.5, color='purple')\n    axes[1,1].set_title('Price vs Taxes Relationship', fontsize=14, fontweight='bold')\n    axes[1,1].set_xlabel('Total Price ($)')\n    axes[1,1].set_ylabel('Taxes ($)')\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # Correlation analysis for numeric columns\n    numeric_cols = ['totalPrice', 'taxes', 'selected']\n    if len(numeric_cols) > 1:\n        print(f\"\\n📊 Correlation Matrix:\")\n        corr_matrix = df[numeric_cols].corr()\n        print(corr_matrix.round(3))\n        \n        plt.figure(figsize=(8, 6))\n        sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', center=0, \n                   square=True, fmt='.3f')\n        plt.title('Feature Correlation Matrix', fontsize=16, fontweight='bold')\n        plt.show()\n    \n    # Selection patterns by price ranges\n    df['price_quartile'] = pd.qcut(df['totalPrice'], 4, labels=['Low', 'Medium-Low', 'Medium-High', 'High'])\n    selection_by_price = df.groupby('price_quartile')['selected'].agg(['count', 'sum', 'mean'])\n    \n    print(f\"\\n💡 Selection Patterns by Price Quartile:\")\n    print(selection_by_price.round(3))\n    \n    return ranker_stats\n\n# Perform comprehensive EDA\nranker_stats = comprehensive_eda(train_df)\n\n# ===================================================================\n# ⚙️ FIXED ADVANCED FEATURE ENGINEERING - HANDLES MISSING FEATURES\n# ===================================================================\n\ndef create_ranking_features(df, is_train=True):\n    \"\"\"\n    Create sophisticated features optimized for flight ranking\n    FIXED: Always creates all features, even if source columns are missing\n    \"\"\"\n    print(f\"\\n🔧 Creating Advanced Ranking Features...\")\n    \n    df_features = df.copy()\n    \n    # === PRICE-BASED FEATURES ===\n    print(\"   💰 Creating price-based features...\")\n    df_features['price_log'] = np.log1p(df_features['totalPrice'])\n    df_features['tax_ratio'] = df_features['taxes'] / (df_features['totalPrice'] + 1)\n    df_features['price_per_tax'] = df_features['totalPrice'] / (df_features['taxes'] + 1)\n    df_features['tax_log'] = np.log1p(df_features['taxes'])\n    \n    # Price percentiles within each ranker group\n    df_features['price_rank_in_group'] = df_features.groupby('ranker_id')['totalPrice'].rank(pct=True)\n    df_features['tax_rank_in_group'] = df_features.groupby('ranker_id')['taxes'].rank(pct=True)\n    \n    # === TIME-BASED FEATURES ===\n    if 'requestDate' in df_features.columns:\n        print(\"   ⏰ Creating time-based features...\")\n        df_features['requestDate'] = pd.to_datetime(df_features['requestDate'], errors='coerce')\n        df_features['request_hour'] = df_features['requestDate'].dt.hour\n        df_features['request_dow'] = df_features['requestDate'].dt.dayofweek\n        df_features['request_month'] = df_features['requestDate'].dt.month\n        \n        # Fill NaN values before converting to int\n        df_features['request_hour'] = df_features['request_hour'].fillna(12).astype(int)\n        df_features['request_dow'] = df_features['request_dow'].fillna(0).astype(int)\n        df_features['request_month'] = df_features['request_month'].fillna(1).astype(int)\n        \n        df_features['is_weekend'] = (df_features['request_dow'] >= 5).astype(int)\n        df_features['is_business_hours'] = ((df_features['request_hour'] >= 9) & \n                                           (df_features['request_hour'] <= 17)).astype(int)\n        df_features['is_morning'] = (df_features['request_hour'] < 12).astype(int)\n        df_features['is_evening'] = (df_features['request_hour'] >= 18).astype(int)\n    else:\n        # Create default time features if requestDate is missing\n        df_features['request_hour'] = 12\n        df_features['request_dow'] = 0\n        df_features['request_month'] = 1\n        df_features['is_weekend'] = 0\n        df_features['is_business_hours'] = 1\n        df_features['is_morning'] = 0\n        df_features['is_evening'] = 0\n    \n    # === ROUTE AND SEARCH FEATURES (FIXED) ===\n    if 'searchRoute' in df_features.columns:\n        print(\"   🛫 Creating route-based features...\")\n        # FIX: Handle NaN values properly before converting to int\n        df_features['is_roundtrip'] = df_features['searchRoute'].str.contains('/', na=False).astype(int)\n        df_features['route_length'] = df_features['searchRoute'].str.len().fillna(0).astype(int)\n        df_features['route_complexity'] = (df_features['searchRoute'].str.count('-').fillna(0) + 1).astype(int)\n    else:\n        # Create default route features if searchRoute is missing\n        df_features['is_roundtrip'] = 0\n        df_features['route_length'] = 0\n        df_features['route_complexity'] = 1\n    \n    # === USER BEHAVIOR FEATURES (FIXED FOR STRING BOOLEANS) ===\n    if all(col in df_features.columns for col in ['isVip', 'bySelf']):\n        print(\"   👤 Creating user behavior features...\")\n        \n        # FIXED: Handle string boolean values properly\n        df_features = fix_boolean_columns(df_features, 'isVip')\n        df_features = fix_boolean_columns(df_features, 'bySelf')\n        \n        df_features['user_vip_score'] = df_features['isVip'] * 2 + df_features['bySelf']\n        df_features['is_vip_self'] = (df_features['isVip'] & df_features['bySelf']).astype(int)\n    else:\n        # Create default user features if columns are missing\n        df_features['isVip'] = 0\n        df_features['bySelf'] = 0\n        df_features['user_vip_score'] = 0\n        df_features['is_vip_self'] = 0\n    \n    # === CORPORATE FEATURES (FIXED) ===\n    if 'corporateTariffCode' in df_features.columns:\n        print(\"   🏢 Creating corporate features...\")\n        df_features['has_corporate_tariff'] = df_features['corporateTariffCode'].notna().astype(int)\n    else:\n        df_features['has_corporate_tariff'] = 0\n    \n    # FIXED: Always create is_tp_compliant feature\n    if 'pricingInfo_isAccessTP' in df_features.columns:\n        df_features['is_tp_compliant'] = df_features['pricingInfo_isAccessTP'].fillna(0).astype(int)\n    else:\n        df_features['is_tp_compliant'] = 0\n    \n    # === SEAT AVAILABILITY FEATURES (FIXED) ===\n    seat_cols = [col for col in df_features.columns if 'seatsAvailable' in col]\n    if seat_cols:\n        print(\"   💺 Creating seat availability features...\")\n        # Handle NaN values in seat columns\n        df_features[seat_cols] = df_features[seat_cols].fillna(0)\n        df_features['min_seats'] = df_features[seat_cols].min(axis=1)\n        df_features['max_seats'] = df_features[seat_cols].max(axis=1)\n        df_features['avg_seats'] = df_features[seat_cols].mean(axis=1)\n        df_features['total_seats'] = df_features[seat_cols].sum(axis=1)\n        df_features['seat_variance'] = df_features[seat_cols].var(axis=1).fillna(0)\n    else:\n        # FIXED: Always create seat features even if source columns are missing\n        print(\"   💺 Creating default seat features (no seat columns found)...\")\n        df_features['min_seats'] = 0\n        df_features['max_seats'] = 0\n        df_features['avg_seats'] = 0\n        df_features['total_seats'] = 0\n        df_features['seat_variance'] = 0\n    \n    # === STATISTICAL FEATURES WITHIN RANKER GROUPS ===\n    print(\"   📊 Creating statistical features within ranker groups...\")\n    \n    # Price statistics within each ranker group\n    price_stats = df_features.groupby('ranker_id')['totalPrice'].agg(['mean', 'std', 'min', 'max']).add_prefix('group_price_')\n    df_features = df_features.merge(price_stats, left_on='ranker_id', right_index=True, how='left')\n    \n    # Handle NaN in group statistics (for single-item groups)\n    df_features['group_price_std'] = df_features['group_price_std'].fillna(0)\n    \n    # Relative price position\n    df_features['price_vs_group_mean'] = df_features['totalPrice'] - df_features['group_price_mean']\n    df_features['price_vs_group_min'] = df_features['totalPrice'] - df_features['group_price_min']\n    df_features['price_zscore_in_group'] = (df_features['totalPrice'] - df_features['group_price_mean']) / (df_features['group_price_std'] + 1e-6)\n    \n    # === INTERACTION FEATURES ===\n    print(\"   🔗 Creating interaction features...\")\n    df_features['price_tax_interaction'] = df_features['totalPrice'] * df_features['tax_ratio']\n    \n    if 'request_hour' in df_features.columns:\n        df_features['price_hour_interaction'] = df_features['price_log'] * df_features['request_hour']\n    \n    print(f\"   ✅ Feature engineering completed. New shape: {df_features.shape}\")\n    \n    return df_features\n\n# ===================================================================\n# 🎯 FIXED DATA PREPARATION\n# ===================================================================\n\ndef prepare_ranking_data(df, target_col='selected', encoders=None, is_train=True):\n    \"\"\"\n    Prepare data specifically optimized for XGBoost Ranker\n    FIXED: Better handling of missing values and data types\n    \"\"\"\n    print(f\"\\n🎯 Preparing Data for XGBoost Ranker...\")\n    \n    df_processed = df.copy()\n    \n    # Select the most predictive features for ranking\n    core_features = [\n        'ranker_id', 'totalPrice', 'taxes', 'price_log', 'tax_ratio', 'price_per_tax',\n        'tax_log', 'price_rank_in_group', 'tax_rank_in_group', 'price_vs_group_mean',\n        'price_vs_group_min', 'price_zscore_in_group', 'price_tax_interaction'\n    ]\n    \n    # Add time features if available\n    time_features = ['request_hour', 'request_dow', 'is_weekend', 'is_business_hours', \n                    'is_morning', 'is_evening', 'price_hour_interaction']\n    for feat in time_features:\n        if feat in df_processed.columns:\n            core_features.append(feat)\n    \n    # Add route features if available\n    route_features = ['is_roundtrip', 'route_length', 'route_complexity']\n    for feat in route_features:\n        if feat in df_processed.columns:\n            core_features.append(feat)\n    \n    # Add user features if available\n    user_features = ['user_vip_score', 'is_vip_self', 'companyID', 'nationality', 'isVip', 'bySelf']\n    for feat in user_features:\n        if feat in df_processed.columns:\n            core_features.append(feat)\n    \n    # Add corporate features if available\n    corp_features = ['has_corporate_tariff', 'is_tp_compliant']\n    for feat in corp_features:\n        if feat in df_processed.columns:\n            core_features.append(feat)\n    \n    # Add seat features if available\n    seat_features = ['min_seats', 'max_seats', 'avg_seats', 'total_seats', 'seat_variance']\n    for feat in seat_features:\n        if feat in df_processed.columns:\n            core_features.append(feat)\n    \n    # Filter to available features\n    available_features = [col for col in core_features if col in df_processed.columns]\n    if target_col in df_processed.columns:\n        available_features.append(target_col)\n    \n    df_processed = df_processed[available_features]\n    \n    print(f\"   📋 Selected {len(available_features)-1 if target_col in available_features else len(available_features)} features for ranking\")\n    \n    # Handle missing values intelligently (IMPROVED)\n    numeric_cols = df_processed.select_dtypes(include=[np.number]).columns\n    for col in numeric_cols:\n        if col != target_col and df_processed[col].isnull().sum() > 0:\n            if 'price' in col.lower() or 'tax' in col.lower():\n                df_processed[col] = df_processed[col].fillna(df_processed[col].median())\n            else:\n                df_processed[col] = df_processed[col].fillna(0)\n    \n    # Handle categorical encoding for ranker_id (CRITICAL for ranking)\n    if encoders is None:\n        encoders = {}\n    \n    if 'ranker_id' in df_processed.columns:\n        if is_train:\n            print(\"   🔢 Encoding ranker_id for training...\")\n            df_processed['ranker_id'] = df_processed['ranker_id'].astype('category')\n            encoders['ranker_id_categories'] = df_processed['ranker_id'].cat.categories\n            df_processed['ranker_id'] = df_processed['ranker_id'].cat.codes\n        else:\n            print(\"   🔢 Encoding ranker_id for testing...\")\n            df_processed['ranker_id'] = df_processed['ranker_id'].astype('category')\n            df_processed['ranker_id'] = df_processed['ranker_id'].cat.set_categories(\n                encoders['ranker_id_categories'], ordered=True)\n            df_processed['ranker_id'] = df_processed['ranker_id'].cat.codes\n    \n    # Handle other categorical variables\n    categorical_cols = df_processed.select_dtypes(include=['object']).columns\n    for col in categorical_cols:\n        if col != target_col:\n            if is_train:\n                le = LabelEncoder()\n                df_processed[col] = le.fit_transform(df_processed[col].astype(str))\n                encoders[f'{col}_encoder'] = le\n            else:\n                if f'{col}_encoder' in encoders:\n                    le = encoders[f'{col}_encoder']\n                    # Handle unseen categories\n                    unique_vals = df_processed[col].unique()\n                    for val in unique_vals:\n                        if val not in le.classes_:\n                            df_processed[col] = df_processed[col].replace(val, 'unknown')\n                    df_processed[col] = le.transform(df_processed[col].astype(str))\n                else:\n                    df_processed[col] = 0\n    \n    print(f\"   ✅ Data preparation completed. Final shape: {df_processed.shape}\")\n    \n    return df_processed, encoders\n\n# Apply advanced feature engineering\ntrain_enhanced = create_ranking_features(train_df, is_train=True)\n\n# Prepare training data\ntrain_processed, encoders = prepare_ranking_data(train_enhanced, is_train=True)\n\n# ===================================================================\n# 🚀 ADVANCED XGBOOST RANKER TRAINING\n# ===================================================================\n\ndef train_advanced_ranker(df, target_col='selected', validation_split=0.2):\n    \"\"\"\n    Train an advanced XGBoost Ranker with proper validation\n    This is the core ranking algorithm that will win the competition\n    \"\"\"\n    print(f\"\\n🚀 Training Advanced XGBoost Ranker...\")\n    \n    # Prepare features and target\n    feature_cols = [col for col in df.columns if col not in [target_col, 'ranker_id']]\n    X = df[['ranker_id'] + feature_cols].copy()\n    y = df[target_col].copy()\n    \n    print(f\"   📊 Training data shape: {X.shape}\")\n    print(f\"   🎯 Target distribution: {y.value_counts().to_dict()}\")\n    \n    # Create group sizes for ranking (ESSENTIAL for XGBoost Ranker)\n    print(\"   🔢 Creating group structure for ranking...\")\n    group_data = X.groupby('ranker_id').size().reset_index(name='group_size')\n    group_sizes = group_data['group_size'].tolist()\n    \n    print(f\"   📈 Ranking groups statistics:\")\n    print(f\"      Total groups: {len(group_sizes)}\")\n    print(f\"      Avg group size: {np.mean(group_sizes):.2f}\")\n    print(f\"      Min group size: {np.min(group_sizes)}\")\n    print(f\"      Max group size: {np.max(group_sizes)}\")\n    \n    # Prepare features (remove ranker_id from features but keep for grouping)\n    X_features = X[feature_cols].copy()\n    \n    # Split data while preserving group structure\n    print(\"   🔄 Creating validation split...\")\n    unique_rankers = X['ranker_id'].unique()\n    train_rankers, val_rankers = train_test_split(\n        unique_rankers, test_size=validation_split, random_state=42\n    )\n    \n    train_mask = X['ranker_id'].isin(train_rankers)\n    val_mask = X['ranker_id'].isin(val_rankers)\n    \n    X_train = X_features[train_mask]\n    y_train = y[train_mask]\n    X_val = X_features[val_mask]\n    y_val = y[val_mask]\n    \n    # Recalculate group sizes for split data\n    train_groups = X[train_mask]['ranker_id'].value_counts().sort_index().tolist()\n    val_groups = X[val_mask]['ranker_id'].value_counts().sort_index().tolist()\n    \n    print(f\"   📊 Training set: {X_train.shape}, Groups: {len(train_groups)}\")\n    print(f\"   📊 Validation set: {X_val.shape}, Groups: {len(val_groups)}\")\n    \n    # Advanced XGBoost Ranker with optimized hyperparameters\n    print(\"   🏆 Initializing XGBoost Ranker with competition-winning parameters...\")\n    \n    ranker = xgb.XGBRanker(\n        objective='rank:pairwise',        # Pairwise ranking for flight preferences\n        n_estimators=1000,                # More trees for complex patterns\n        max_depth=8,                      # Deep trees for feature interactions\n        learning_rate=0.01,               # Conservative learning rate\n        subsample=0.8,                    # Prevent overfitting\n        colsample_bytree=0.8,            # Feature sampling\n        reg_alpha=0.1,                    # L1 regularization\n        reg_lambda=0.2,                   # L2 regularization\n        random_state=42,\n        n_jobs=-1,                        # Use all CPU cores\n        tree_method='hist',               # Faster training\n        eval_metric='ndcg@3',             # Optimize for top-3 ranking (competition metric)\n        early_stopping_rounds=50,         # Stop if no improvement\n        verbosity=1\n    )\n    \n    # Train with group information (CRITICAL for ranking)\n    print(\"   🔥 Training ranker with group structure...\")\n    \n    ranker.fit(\n        X_train, y_train, \n        group=train_groups,               # Essential group parameter\n        eval_set=[(X_val, y_val)],\n        eval_group=[val_groups],\n        verbose=True\n    )\n    \n    print(\"   ✅ Training completed!\")\n    \n    # Validation predictions and evaluation\n    print(\"\\n   📊 Validation Performance:\")\n    val_scores = ranker.predict(X_val)\n    \n    # Calculate NDCG@3 for validation (competition metric)\n    val_ndcg_scores = []\n    val_ranker_ids = X[val_mask]['ranker_id'].values\n    \n    for ranker_id in np.unique(val_ranker_ids):\n        mask = val_ranker_ids == ranker_id\n        if np.sum(mask) > 1:  # Need at least 2 items to rank\n            group_y_true = y_val[mask].values.reshape(1, -1)\n            group_y_scores = val_scores[mask].reshape(1, -1)\n            ndcg = ndcg_score(group_y_true, group_y_scores, k=3)\n            val_ndcg_scores.append(ndcg)\n    \n    avg_ndcg = np.mean(val_ndcg_scores) if val_ndcg_scores else 0\n    print(f\"      Average NDCG@3: {avg_ndcg:.4f}\")\n    print(f\"      NDCG@3 std: {np.std(val_ndcg_scores):.4f}\" if val_ndcg_scores else \"      NDCG@3 std: 0.0000\")\n    \n    # Feature importance analysis\n    print(\"\\n   🔍 Top 10 Most Important Features:\")\n    feature_importance = ranker.feature_importances_\n    feature_names = X_features.columns\n    \n    importance_df = pd.DataFrame({\n        'feature': feature_names,\n        'importance': feature_importance\n    }).sort_values('importance', ascending=False)\n    \n    print(importance_df.head(10).to_string(index=False))\n    \n    # Plot feature importance\n    plt.figure(figsize=(12, 8))\n    top_features = importance_df.head(15)\n    plt.barh(range(len(top_features)), top_features['importance'], color='skyblue')\n    plt.yticks(range(len(top_features)), top_features['feature'])\n    plt.xlabel('Feature Importance')\n    plt.title('Top 15 Most Important Features for Flight Ranking', fontsize=16, fontweight='bold')\n    plt.gca().invert_yaxis()\n    plt.tight_layout()\n    plt.show()\n    \n    return ranker, X_features.columns, group_sizes, avg_ndcg\n\n# Train the advanced ranker\nranker_model, feature_columns, original_group_sizes, validation_ndcg = train_advanced_ranker(train_processed)\n\n# ===================================================================\n# 🎯 FIXED PREDICTION GENERATION - HANDLES MISSING FEATURES\n# ===================================================================\n\ndef generate_competition_predictions(model, feature_cols, encoders, sample_submission, train_df):\n    \"\"\"\n    Generate final predictions for competition submission\n    FIXED: Handles missing features by creating them with default values\n    \"\"\"\n    print(f\"\\n🎯 Generating Competition Predictions...\")\n    \n    # Create test features from sample submission\n    test_data = sample_submission[['Id', 'ranker_id']].copy()\n    print(f\"   📋 Test data shape: {test_data.shape}\")\n    \n    # Merge with training data to get feature information\n    print(\"   🔗 Merging with training features...\")\n    \n    # Get unique combinations from training data\n    feature_source = train_df.groupby('ranker_id').agg({\n        'taxes': 'first',\n        'totalPrice': 'first',\n        'companyID': 'first',\n        'nationality': 'first', \n        'isVip': 'first',\n        'bySelf': 'first',\n        'requestDate': 'first',\n        'searchRoute': 'first',\n        'corporateTariffCode': 'first'\n    }).reset_index()\n    \n    test_enhanced = test_data.merge(feature_source, on='ranker_id', how='left')\n    print(f\"   📊 Enhanced test data shape: {test_enhanced.shape}\")\n    \n    # FIXED: Handle missing values based on data types\n    print(\"   🔧 Handling missing values by data type...\")\n    \n    # Handle each column based on its data type\n    test_enhanced['taxes'] = safe_fillna_by_dtype(test_enhanced['taxes'], train_df['taxes'].median())\n    test_enhanced['totalPrice'] = safe_fillna_by_dtype(test_enhanced['totalPrice'], train_df['totalPrice'].median())\n    \n    # For categorical columns, use string fill values\n    test_enhanced['companyID'] = safe_fillna_by_dtype(test_enhanced['companyID'], 'unknown')\n    test_enhanced['nationality'] = safe_fillna_by_dtype(test_enhanced['nationality'], 'unknown')\n    test_enhanced['searchRoute'] = safe_fillna_by_dtype(test_enhanced['searchRoute'], 'unknown')\n    test_enhanced['corporateTariffCode'] = safe_fillna_by_dtype(test_enhanced['corporateTariffCode'], 'unknown')\n    \n    # FIXED: For boolean columns that might be strings\n    # Handle isVip and bySelf with string boolean fix\n    if 'isVip' in test_enhanced.columns:\n        test_enhanced['isVip'] = test_enhanced['isVip'].fillna('False')\n    if 'bySelf' in test_enhanced.columns:\n        test_enhanced['bySelf'] = test_enhanced['bySelf'].fillna('False')\n    \n    # For datetime columns\n    if 'requestDate' in test_enhanced.columns:\n        mode_date = train_df['requestDate'].mode()\n        default_date = mode_date.iloc[0] if len(mode_date) > 0 else '2024-01-01'\n        test_enhanced['requestDate'] = test_enhanced['requestDate'].fillna(default_date)\n    \n    # Apply same feature engineering\n    print(\"   ⚙️ Applying feature engineering...\")\n    test_enhanced = create_ranking_features(test_enhanced, is_train=False)\n    \n    # Apply same preprocessing\n    print(\"   🔧 Applying preprocessing...\")\n    test_processed, _ = prepare_ranking_data(test_enhanced, encoders=encoders, is_train=False)\n    \n    # FIXED: Handle missing features by creating them with default values\n    print(\"   🔧 Ensuring all required features are present...\")\n    for col in feature_cols:\n        if col not in test_processed.columns:\n            print(f\"      Creating missing feature: {col}\")\n            test_processed[col] = 0\n    \n    # Select only the required features in the correct order\n    test_features = test_processed[feature_cols].copy()\n    \n    print(f\"   📊 Final test features shape: {test_features.shape}\")\n    \n    # Generate predictions (scores)\n    print(\"   🔮 Generating model predictions...\")\n    test_scores = model.predict(test_features)\n    \n    # Convert scores to ranks within each ranker_id group\n    print(\"   🏆 Converting scores to rankings...\")\n    submission = sample_submission.copy()\n    \n    # Group by ranker_id and convert scores to ranks\n    def scores_to_ranks_within_group(group_indices, scores_dict):\n        \"\"\"Convert scores to ranks within each group (1 = best)\"\"\"\n        group_scores = [scores_dict[idx] for idx in group_indices]\n        # Higher scores get lower ranks (1 = best, 2 = second best, etc.)\n        ranks = np.argsort(np.argsort(-np.array(group_scores))) + 1\n        return ranks\n    \n    # Create score dictionary\n    score_dict = dict(zip(test_data.index, test_scores))\n    \n    # Apply ranking within each group\n    print(\"   🎯 Applying within-group ranking...\")\n    final_ranks = []\n    \n    for ranker_id in submission['ranker_id'].unique():\n        group_mask = submission['ranker_id'] == ranker_id\n        group_indices = submission[group_mask].index.tolist()\n        group_ranks = scores_to_ranks_within_group(group_indices, score_dict)\n        \n        for idx, rank in zip(group_indices, group_ranks):\n            final_ranks.append((idx, rank))\n    \n    # Sort by original index and assign ranks\n    final_ranks.sort(key=lambda x: x[0])\n    submission['selected'] = [rank for _, rank in final_ranks]\n    \n    return submission, test_scores\n\n# Generate final predictions\nfinal_submission, prediction_scores = generate_competition_predictions(\n    ranker_model, feature_columns, encoders, sample_submission, train_df\n)\n\n# ===================================================================\n# ✅ VALIDATION AND SUBMISSION\n# ===================================================================\n\nprint(f\"\\n✅ FINAL VALIDATION AND SUBMISSION\")\nprint(\"=\"*60)\n\n# Validate submission format\nprint(f\"📊 Submission Validation:\")\nprint(f\"   Submission shape: {final_submission.shape}\")\nprint(f\"   Expected shape: {sample_submission.shape}\")\nprint(f\"   Columns match: {list(final_submission.columns) == list(sample_submission.columns)}\")\n\n# Check ranking validity for each group\nprint(f\"\\n🔍 Ranking Validation:\")\nranking_errors = 0\ntotal_groups = 0\n\nfor ranker_id in final_submission['ranker_id'].unique():\n    group = final_submission[final_submission['ranker_id'] == ranker_id]['selected']\n    expected_ranks = set(range(1, len(group) + 1))\n    actual_ranks = set(group.values)\n    \n    total_groups += 1\n    if expected_ranks != actual_ranks:\n        ranking_errors += 1\n        if ranking_errors <= 3:  # Show first few errors\n            print(f\"   ⚠️  Invalid ranks for ranker_id {ranker_id}: {sorted(actual_ranks)} vs expected {sorted(expected_ranks)}\")\n\nprint(f\"   Total groups checked: {total_groups}\")\nprint(f\"   Groups with ranking errors: {ranking_errors}\")\nprint(f\"   Ranking validation: {'✅ PASSED' if ranking_errors == 0 else '❌ FAILED'}\")\n\n# Submission statistics\nprint(f\"\\n📈 Submission Statistics:\")\nprint(f\"   Unique ranker_ids: {final_submission['ranker_id'].nunique()}\")\nprint(f\"   Rank distribution:\")\nrank_dist = final_submission['selected'].value_counts().sort_index()\nfor rank, count in rank_dist.head(10).items():\n    print(f\"      Rank {rank}: {count} flights\")\n\nprint(f\"\\n🎯 Model Performance Summary:\")\nprint(f\"   Validation NDCG@3: {validation_ndcg:.4f}\")\nprint(f\"   Features used: {len(feature_columns)}\")\nprint(f\"   Training groups: {len(original_group_sizes)}\")\n\n# Final submission preview\nprint(f\"\\n📋 Final Submission Preview:\")\nprint(final_submission.head(15))\n\n# Save submission\nfinal_submission.to_parquet('advanced_xgboost_ranker_submission.parquet', index=False)\nprint(f\"\\n🏆 COMPETITION SUBMISSION SAVED!\")\nprint(f\"   File: advanced_xgboost_ranker_submission.parquet\")\nprint(f\"   This XGBoost Ranker approach should significantly improve your leaderboard position!\")\nprint(f\"   The model optimizes for HitRate@3 which is exactly what the competition evaluates.\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"🎉 ADVANCED XGBOOST RANKER PIPELINE COMPLETED SUCCESSFULLY!\")\nprint(\"=\"*60)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T12:28:56.600725Z","iopub.execute_input":"2025-06-22T12:28:56.601013Z","execution_failed":"2025-06-22T14:02:04.820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}