{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U xgboost\n!pip install -U polars\n!pip install -U optuna\n!pip install -U catboost\n!pip install -U lightgbm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nimport catboost\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import ndcg_score\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_hubs = (\n    train.select(\"legs0_segments1_departureFrom_airport_iata\")\n         .drop_nulls()\n         .group_by(\"legs0_segments1_departureFrom_airport_iata\")\n         .agg(pl.count().alias(\"count\"))\n         .sort(\"count\", descending=True)\n         .head(10)\n)\n\nhub_airports = top_hubs[\"legs0_segments1_departureFrom_airport_iata\"].to_list()\nhub_airports","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\n\n# Combine all initial transformations\ndf = df.with_columns([\n        # Price features\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # Duration features\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n        \n        # Trip type\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # Total segments count\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # FF features\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # Binary features\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n\n        (\n            (pl.col(\"miniRules0_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules0_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_cancel\"),\n        (\n            (pl.col(\"miniRules1_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules1_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_exchange\"),\n    \n        # Routes & carriers\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # Cabin\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n])\n\n# Segment counts - more efficient\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n    else:\n        seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n# Add segment-based features\n# First create segment counts\ndf = df.with_columns(seg_exprs)\n\n# Then use them for derived features\ndf = df.with_columns([\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n])\n\n# More derived features\ndf = df.with_columns([\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n])\n\n# Add major carrier flag if column exists\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns(\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\")\n    )\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\ndf = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n# Time features - batch process\ntime_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# Batch rank computations - more efficient with single pass\n# First apply the columns that will be used for ranking\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n])\n\n# Price and duration basic ranks\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n# Price-specific features\nprice_exprs = [\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n]\n\n# Apply initial ranks\ndf = df.with_columns(rank_exprs + price_exprs)\n\n# Cheapest direct - more efficient\ndirect_cheapest = (\n    df.filter(pl.col(\"is_direct_leg0\") == 1)\n    .group_by(\"ranker_id\")\n    .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n)\n\ndf = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n    ((pl.col(\"is_direct_leg0\") == 1) & \n     (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n).drop(\"min_direct\")\n\n# Popularity features - efficient join\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n    ])\n)\n\n# Step 1: Add independent features\ndf = df.with_columns([\n    # Carrier popularity\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n])\n\ndf = df.with_columns(\n    (\n        # Policy compliance (25% weight)\n        (pl.col(\"pricingInfo_isAccessTP\") * 0.25) +\n        # Direct flights (25% weight)\n        (pl.col(\"is_direct_leg0\") * 0.25) +\n        # Business-hour departures/arrivals (25% weight)\n        ((pl.col(\"legs0_departureAt_business_time\") + pl.col(\"legs1_departureAt_business_time\")) * 0.125) +\n        # VIP preference for business class (25% weight)\n        ((pl.col(\"isVip\") == 1) & (pl.col(\"avg_cabin_class\") >= 1.5)).cast(pl.Int8) * 0.25\n    ).alias(\"business_traveler_perfect_match\"),\n\n    # Timezone diff only\n    (pl.col(\"legs0_arrivalAt_hour\") - pl.col(\"legs0_departureAt_hour\") -\n     (pl.col(\"legs0_duration\") / 60)).alias(\"timezone_diff_leg0\"),\n)\n\ndf = df.with_columns(\n    (\n        (pl.col(\"is_one_way\") == 0) &  # Round-trip\n        (pl.col(\"legs0_arrivalAt_hour\") >= 8) &  # Arrive by morning\n        (pl.col(\"legs1_departureAt_hour\") <= 18) &  # Return by evening\n        (pl.col(\"timezone_diff_leg0\").abs() < 3)   # Minimal jetlag\n    ).cast(pl.Int8).alias(\"meeting_friendly_itinerary\")\n)\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = df.with_columns(\n    [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n    [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber'\n]\n\n# Columns to exclude (uninformative or problematic)\nexclude_cols = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n    'frequentFlyer',  # Already processed\n    # Exclude constant columns\n    'pricingInfo_passengerCount','bySelf','n_segments_leg1','timezone_diff_leg0','meeting_friendly_itinerary','business_traveler_perfect_match'\n]\n\nfor leg in [0, 1]:\n    for seg in [0, 1]:\n        if seg == 0:\n            suffixes = [\n                \"seatsAvailable\",\n            ]\n        else:\n            suffixes = [\n                \"cabinClass\",\n                \"seatsAvailable\",\n                \"baggageAllowance_quantity\",\n                \"baggageAllowance_weightMeasurementType\",\n                \"aircraft_code\",\n                \"arrivalTo_airport_city_iata\",\n                \"arrivalTo_airport_iata\",\n                \"departureFrom_airport_iata\",\n                \"flightNumber\",\n                \"marketingCarrier_code\",\n                \"operatingCarrier_code\",\n            ]\n        for suffix in suffixes:\n            exclude_cols.append(f\"legs{leg}_segments{seg}_{suffix}\")\n\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols + ['ranker_id'])\ny = data.select(['selected', 'ranker_id'])\ngroups = data.select('ranker_id')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb = X.with_columns([(pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int16) for c in cat_features_final])\n\nn1 = 16487352 # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n2], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n2], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n2], groups[n1:n2], groups[n2:]\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\ngroup_sizes_tr_lgb = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_va_lgb = groups_va.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_te_lgb = groups_te.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Step 1: group size & median ---\nranker_group_sizes = groups_tr.group_by('ranker_id').agg(pl.len().alias('group_size'))\nmedian_group_size = ranker_group_sizes['group_size'].median()\nprint(f\"Median group size: {median_group_size}\")\n\n# --- Step 2: get ranker_id lists ---\nsmall_rankers = ranker_group_sizes.filter(pl.col('group_size') < median_group_size)['ranker_id']\nbig_rankers   = ranker_group_sizes.filter(pl.col('group_size') >= median_group_size)['ranker_id']\n\n# --- Step 3: rankers where is_min_segments == 0 ---\nmin_segment_0 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 0)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\n\n\n# --- Step 4: filtering helper ---\ndef split_data(ranker_ids):\n    ranker_list = ranker_ids.to_list() if hasattr(ranker_ids, \"to_list\") else list(ranker_ids)\n    data = data_xgb_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    y    = y_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    grp  = groups_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    return (\n        data.drop('ranker_id').to_pandas(),\n        y.drop('ranker_id')['selected'].to_numpy(),\n        grp\n    )\n\n# --- Step 5: create splits ---\ndata_xgb_tr_small_pd, y_tr_small_pd, groups_tr_small   = split_data(small_rankers)\ndata_xgb_tr_big_pd,   y_tr_big_pd,   groups_tr_big     = split_data(big_rankers)\ndata_xgb_tr_min0_pd,  y_tr_min0_pd,  groups_tr_min0    = split_data(min_segment_0)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb_tr.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import ndcg_score\n\n# =============================================================================\n# CONSTANTS AND GLOBALS\n# =============================================================================\nRANDOM_STATE = 42\nshap_lgbm_seed = {}\nxgb_seeds = [50]\nlgb_seeds = [42]\nall_models = []\nfeature_importances = []\n\n# =============================================================================\n# XGBOOST PARAMETERS\n# =============================================================================\nxgb_rank_params = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_small = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 22,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_big = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_min0 = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\n# Use all data splits\nxgb_params = {\n    'full': xgb_rank_params,\n    'small': xgb_rank_params_small, \n    'big': xgb_rank_params_big,\n    'min0': xgb_rank_params_min0\n}\n\n# =============================================================================\n# LIGHTGBM PARAMETERS\n# =============================================================================\nlgb_rank_params = {\n    'objective': 'lambdarank',\n    'metric': 'ndcg',\n    'boosting_type': 'gbdt',\n    'eval_at': [3],\n    'num_leaves': 137,\n    'learning_rate': 0.1923609,\n    'min_child_samples': 69,\n    'lambda_l1': 0.0017863,\n    'lambda_l2': 7.8818,\n    'feature_fraction': 0.6015,\n    'bagging_fraction': 0.8536,\n    'bagging_freq': 7,\n    'verbosity': -1,\n    'label_gain': [0, 1]\n}\n\n# =============================================================================\n# DATA PREPARATION\n# =============================================================================\n# Drop ranker_id from pandas frames\nfor df in [data_xgb_tr_small_pd, data_xgb_tr_big_pd]:\n    if 'ranker_id' in df.columns:\n        df.drop(columns=['ranker_id'], inplace=True)\n\n# Group sizes for small/big/min0\ngroup_sizes_tr_small = groups_tr_small.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_big = groups_tr_big.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_min_0 = groups_tr_min0.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\n# =============================================================================\n# DMATRIX SETUP\n# =============================================================================\ndtrain_full = xgb.DMatrix(\n    data_xgb_tr.drop('ranker_id').to_pandas(),\n    label=y_tr['selected'].to_numpy(),\n    group=group_sizes_tr,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtrain_small = xgb.DMatrix(\n    data_xgb_tr_small_pd,\n    label=y_tr_small_pd,\n    group=group_sizes_tr_small,\n    feature_names=list(data_xgb_tr_small_pd.columns)\n)\n\ndtrain_big = xgb.DMatrix(\n    data_xgb_tr_big_pd,\n    label=y_tr_big_pd,\n    group=group_sizes_tr_big,\n    feature_names=list(data_xgb_tr_big_pd.columns)\n)\n\ndtrain_min_0 = xgb.DMatrix(\n    data_xgb_tr_min0_pd,\n    label=y_tr_min0_pd,\n    group=group_size_tr_min_0,\n    feature_names=list(data_xgb_tr_min0_pd.columns)\n)\n\ndval = xgb.DMatrix(\n    data_xgb_va.drop('ranker_id').to_pandas(),\n    label=y_va['selected'].to_numpy(),\n    group=group_sizes_va,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtest = xgb.DMatrix(\n    data_xgb_te.drop('ranker_id').to_pandas(),\n    label=y_te['selected'].to_numpy(),\n    group=group_sizes_te,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\n# =============================================================================\n# TRAIN XGBOOST MODELS\n# =============================================================================\nprint(\"🚀 Training XGBoost models on all data splits...\")\n\nfor type_of_portion in xgb_params:\n    print(f\"\\nTraining XGBoost model with portion: {type_of_portion}...\")\n    \n    if type_of_portion == 'full':\n        train = dtrain_full\n        val = dval\n    elif type_of_portion == 'small':\n        train = dtrain_small\n        val = dval\n    elif type_of_portion == 'big':\n        train = dtrain_big\n        val = dval\n    elif type_of_portion == 'min0':\n        train = dtrain_min_0\n        val = dval\n    \n    params = xgb_params[type_of_portion].copy()\n    params['seed'] = RANDOM_STATE\n    \n    model = xgb.train(\n        params,\n        train,\n        num_boost_round=600,\n        evals=[(train, 'train'), (val, 'val')],\n        verbose_eval=100\n    )\n    \n    all_models.append(('xgb', RANDOM_STATE, type_of_portion, model))\n    \n    # Feature importance\n    xgb_fi = pd.DataFrame.from_dict(\n        model.get_score(importance_type='gain'),\n        orient='index',\n        columns=['importance_gain']\n    ).reset_index()\n    xgb_fi.columns = ['feature', 'importance_gain']\n    xgb_fi['importance_split'] = list(model.get_score(importance_type='weight').values())\n    xgb_fi['seed'] = RANDOM_STATE\n    xgb_fi['model_type'] = 'xgb'\n    xgb_fi['portion'] = type_of_portion\n    feature_importances.append(xgb_fi)\n\n# =============================================================================\n# TRAIN LIGHTGBM MODELS\n# =============================================================================\nprint(\"\\n🔥 Training LightGBM models...\")\n\nfor seed in lgb_seeds:\n    print(f\"\\nTraining LightGBM model with seed {seed}...\")\n    \n    lgb_train = lgb.Dataset(\n        data=data_xgb_tr.drop('ranker_id').to_pandas(),\n        label=y_tr['selected'].to_numpy(),\n        group=group_sizes_tr_lgb,\n        feature_name=data_xgb.drop('ranker_id').columns,\n        free_raw_data=False\n    )\n    \n    lgb_val = lgb.Dataset(\n        data=data_xgb_va.drop('ranker_id').to_pandas(),\n        label=y_va['selected'].to_numpy(),\n        group=group_sizes_va_lgb,\n        feature_name=data_xgb.drop('ranker_id').columns,\n        reference=lgb_train,\n        free_raw_data=False\n    )\n    \n    params = lgb_rank_params.copy()\n    params['seed'] = seed\n    \n    model = lgb.train(\n        params,\n        lgb_train,\n        num_boost_round=700,\n        valid_sets=[lgb_train, lgb_val],\n        callbacks=[lgb.early_stopping(300), lgb.log_evaluation(100)]\n    )\n    \n    all_models.append(('lgb', seed, model))\n    \n    fi_df = pd.DataFrame({\n        'feature': data_xgb.drop('ranker_id').columns,\n        'importance_split': model.feature_importance(importance_type='split'),\n        'importance_gain': model.feature_importance(importance_type='gain'),\n        'seed': seed,\n        'model_type': 'lgb'\n    })\n    feature_importances.append(fi_df)\n\n# =============================================================================\n# BAGGING MODELS FOR DIVERSITY\n# =============================================================================\ndef train_bagged_xgb_models(n_bags=5):\n    \"\"\"Train XGBoost models on random subsets\"\"\"\n    bagged_models = []\n    print(f\"\\n🎲 Training {n_bags} bagged XGBoost models...\")\n    \n    for bag in range(n_bags):\n        # Random subset với group preservation\n        unique_groups = groups_tr.select('ranker_id').unique()['ranker_id'].to_list()\n        np.random.seed(42 + bag)\n        selected_groups = np.random.choice(unique_groups, size=int(0.85 * len(unique_groups)), replace=False)\n        \n        # Filter data\n        mask = groups_tr.filter(pl.col('ranker_id').is_in(selected_groups))\n        bag_data = data_xgb_tr.filter(pl.col('ranker_id').is_in(selected_groups))\n        bag_y = y_tr.filter(pl.col('ranker_id').is_in(selected_groups))\n        bag_groups = mask.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n        \n        # XGBoost bag\n        dtrain_bag = xgb.DMatrix(\n            bag_data.drop('ranker_id').to_pandas(),\n            label=bag_y['selected'].to_numpy(),\n            group=bag_groups,\n            feature_names=data_xgb.drop('ranker_id').columns\n        )\n        \n        xgb_bag = xgb.train(\n            {**xgb_rank_params, 'seed': 42 + bag, 'learning_rate': 0.025},\n            dtrain_bag,\n            num_boost_round=500,\n            verbose_eval=False\n        )\n        \n        bagged_models.append(('xgb_bag', 42 + bag, f'bag_{bag}', xgb_bag))\n        print(f\"   Bag {bag+1}/{n_bags} completed\")\n    \n    return bagged_models\n\ndef train_bagged_lgb_models(n_bags=3):\n    \"\"\"Train LightGBM models with different random seeds\"\"\"\n    bagged_models = []\n    print(f\"\\n🎲 Training {n_bags} bagged LightGBM models...\")\n    \n    for bag in range(n_bags):\n        lgb_train_bag = lgb.Dataset(\n            data=data_xgb_tr.drop('ranker_id').to_pandas(),\n            label=y_tr['selected'].to_numpy(),\n            group=group_sizes_tr_lgb,\n            feature_name=data_xgb.drop('ranker_id').columns,\n            free_raw_data=False\n        )\n        \n        params_bag = lgb_rank_params.copy()\n        params_bag.update({\n            'seed': 100 + bag,\n            'learning_rate': 0.18 + bag * 0.01,\n            'num_leaves': 137 + bag * 5\n        })\n        \n        lgb_bag = lgb.train(\n            params_bag,\n            lgb_train_bag,\n            num_boost_round=600,\n        )\n        \n        bagged_models.append(('lgb_bag', 100 + bag, lgb_bag))\n        print(f\"   LGB Bag {bag+1}/{n_bags} completed\")\n    \n    return bagged_models\n\n# Train bagged models\nbagged_xgb = train_bagged_xgb_models(5)\nbagged_lgb = train_bagged_lgb_models(3)\nall_models.extend(bagged_xgb)\nall_models.extend(bagged_lgb)\n\nprint(f\"\\n✅ Total models trained: {len(all_models)}\")\n\n# =============================================================================\n# ADVANCED ENSEMBLE TECHNIQUES\n# =============================================================================\n\n# Separate models by type\nlgb_models = []\nxgb_models = []\n\nfor model_info in all_models:\n    if len(model_info) == 3 and model_info[0].startswith('lgb'):  # LGB models\n        lgb_models.append(model_info[2])\n    elif len(model_info) == 4 and model_info[0].startswith('xgb'):  # XGB models  \n        xgb_models.append(model_info[3])\n\nprint(f\"📊 LightGBM models: {len(lgb_models)}\")\nprint(f\"📊 XGBoost models: {len(xgb_models)}\")\n\ndef calculate_model_weights_by_performance():\n    \"\"\"Calculate weights based on individual model performance\"\"\"\n    print(\"\\n📊 Calculating individual model weights...\")\n    \n    lgb_scores = []\n    xgb_scores = []\n    y_true = y_va['selected'].to_numpy().flatten()\n    \n    # LightGBM individual performance\n    for i, model in enumerate(lgb_models):\n        pred = model.predict(data_xgb_va.drop('ranker_id').to_pandas())\n        score = ndcg_score([y_true], [pred], k=3)\n        lgb_scores.append(score)\n        print(f\"   LGB Model {i}: {score:.4f}\")\n    \n    # XGBoost individual performance  \n    for i, model in enumerate(xgb_models):\n        pred = model.predict(dval)\n        score = ndcg_score([y_true], [pred], k=3)\n        xgb_scores.append(score)\n        print(f\"   XGB Model {i}: {score:.4f}\")\n    \n    # Calculate weights (square to emphasize better models)\n    if lgb_scores:\n        lgb_weights = [max(score**2, 0.001) for score in lgb_scores]\n        lgb_weights = [w/sum(lgb_weights) for w in lgb_weights]\n        print(f\"🎯 LGB weights: {[f'{w:.3f}' for w in lgb_weights]}\")\n    else:\n        lgb_weights = []\n    \n    if xgb_scores:\n        xgb_weights = [max(score**2, 0.001) for score in xgb_scores]\n        xgb_weights = [w/sum(xgb_weights) for w in xgb_weights]\n        print(f\"🎯 XGB weights: {[f'{w:.3f}' for w in xgb_weights]}\")\n    else:\n        xgb_weights = []\n    \n    return lgb_weights, xgb_weights, lgb_scores, xgb_scores\n\ndef optimize_blending_weights(lgb_preds, xgb_preds, y_true, n_trials=21):\n    \"\"\"Find optimal blending weights between LGB and XGB\"\"\"\n    best_score = 0\n    best_weight = 0.5\n    \n    print(\"\\n🔍 Optimizing blending weights...\")\n    for alpha in np.linspace(0.0, 1.0, n_trials):\n        blended_pred = alpha * lgb_preds + (1 - alpha) * xgb_preds\n        score = ndcg_score([y_true], [blended_pred], k=3)\n        \n        if score > best_score:\n            best_score = score\n            best_weight = alpha\n        \n        if abs(alpha - round(alpha, 1)) < 0.01:  # Print every 10%\n            print(f\"   Alpha {alpha:.1f}: {score:.4f}\")\n    \n    print(f\"🏆 Best alpha: {best_weight:.3f}, Score: {best_score:.4f}\")\n    return best_weight, best_score\n\n# Calculate optimal weights\nlgb_weights, xgb_weights, lgb_individual_scores, xgb_individual_scores = calculate_model_weights_by_performance()\n\n# Weighted predictions on validation\nprint(\"\\n🔄 Creating ensemble predictions...\")\n\nif lgb_models and lgb_weights:\n    preds_val_lgb_weighted = np.average(\n        [model.predict(data_xgb_va.drop('ranker_id').to_pandas()) for model in lgb_models], \n        axis=0, \n        weights=lgb_weights\n    )\nelse:\n    preds_val_lgb_weighted = np.zeros(len(y_va))\n\nif xgb_models and xgb_weights:\n    preds_val_xgb_weighted = np.average(\n        [model.predict(dval) for model in xgb_models], \n        axis=0, \n        weights=xgb_weights\n    )\nelse:\n    preds_val_xgb_weighted = np.zeros(len(y_va))\n\n# Simple average baseline\nif lgb_models:\n    preds_val_lgb_simple = np.mean([model.predict(data_xgb_va.drop('ranker_id').to_pandas()) for model in lgb_models], axis=0)\nelse:\n    preds_val_lgb_simple = np.zeros(len(y_va))\n\nif xgb_models:\n    preds_val_xgb_simple = np.mean([model.predict(dval) for model in xgb_models], axis=0)\nelse:\n    preds_val_xgb_simple = np.zeros(len(y_va))\n\ny_true = y_va['selected'].to_numpy().flatten()\n\n# Evaluate different ensemble methods\nensemble_methods = {}\n\n# 1. Simple XGB only\nif len(preds_val_xgb_simple) > 0:\n    score_xgb_simple = ndcg_score([y_true], [preds_val_xgb_simple], k=3)\n    ensemble_methods['xgb_simple'] = (preds_val_xgb_simple, score_xgb_simple)\n    print(f\"📊 XGB Simple Average: {score_xgb_simple:.4f}\")\n\n# 2. Weighted XGB only\nif len(preds_val_xgb_weighted) > 0:\n    score_xgb_weighted = ndcg_score([y_true], [preds_val_xgb_weighted], k=3)\n    ensemble_methods['xgb_weighted'] = (preds_val_xgb_weighted, score_xgb_weighted)\n    print(f\"📊 XGB Weighted: {score_xgb_weighted:.4f}\")\n\n# 3. Blending XGB + LGB (if both exist)\nif lgb_models and xgb_models:\n    optimal_alpha, best_blend_score = optimize_blending_weights(\n        preds_val_lgb_weighted, \n        preds_val_xgb_weighted, \n        y_true\n    )\n    preds_val_blended = optimal_alpha * preds_val_lgb_weighted + (1 - optimal_alpha) * preds_val_xgb_weighted\n    ensemble_methods['blended'] = (preds_val_blended, best_blend_score)\n    print(f\"🎯 Blended (α={optimal_alpha:.3f}): {best_blend_score:.4f}\")\n\n# Find best method\nbest_method = max(ensemble_methods.items(), key=lambda x: x[1][1])\nbest_method_name, (best_preds, best_score) = best_method\n\nprint(f\"\\n🏆 BEST ENSEMBLE METHOD: {best_method_name}\")\nprint(f\"🎯 Best Validation Score: {best_score:.4f}\")\n\nif 'xgb_simple' in ensemble_methods:\n    print(f\"📈 Improvement over simple XGB: +{best_score - ensemble_methods['xgb_simple'][1]:.4f}\")\n\n# =============================================================================\n# APPLY BEST METHOD TO TEST SET\n# =============================================================================\nprint(f\"\\n🚀 Applying {best_method_name} to test set...\")\n\nif best_method_name == 'xgb_weighted':\n    ensemble_test_preds = np.average(\n        [model.predict(dtest) for model in xgb_models], \n        axis=0, weights=xgb_weights\n    )\nelif best_method_name == 'xgb_simple':\n    ensemble_test_preds = np.mean([model.predict(dtest) for model in xgb_models], axis=0)\nelif best_method_name == 'blended':\n    preds_test_lgb_base = np.average(\n        [model.predict(data_xgb_te.drop('ranker_id').to_pandas()) for model in lgb_models], \n        axis=0, weights=lgb_weights\n    )\n    preds_test_xgb_base = np.average(\n        [model.predict(dtest) for model in xgb_models], \n        axis=0, weights=xgb_weights\n    )\n    ensemble_test_preds = optimal_alpha * preds_test_lgb_base + (1 - optimal_alpha) * preds_test_xgb_base\n\nprint(f\"✅ Test predictions generated using {best_method_name}\")\n\n# =============================================================================\n# FEATURE IMPORTANCE ANALYSIS\n# =============================================================================\nall_feature_importance = pd.concat(feature_importances, ignore_index=True)\n\n# Group by feature and model_type\nagg_importance = (\n    all_feature_importance\n    .groupby([\"model_type\", \"feature\"])[[\"importance_split\", \"importance_gain\"]]\n    .mean()\n    .reset_index()\n)\n\n# Separate and sort LightGBM and XGBoost\nif 'lgb' in agg_importance['model_type'].values:\n    lgb_fi_sorted = (\n        agg_importance[agg_importance[\"model_type\"] == \"lgb\"]\n        .sort_values(\"importance_gain\", ascending=False)\n    )\n\nxgb_fi_sorted = (\n    agg_importance[agg_importance[\"model_type\"] == \"xgb\"]\n    .sort_values(\"importance_gain\", ascending=False)\n)\n\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\npd.set_option('display.width', 0)\npd.set_option('display.max_colwidth', None)\n\nprint(\"\\n🔝 Top XGBoost Features (by Gain):\")\nprint(xgb_fi_sorted.head(50))\n\n# =============================================================================\n# FINAL EVALUATION AND SUBMISSION\n# =============================================================================\n\n# HitRate evaluation (if function exists)\ntry:\n    ensemble_hr3 = hitrate_at_3(y_true, best_preds, groups_va['ranker_id'])\n    print(f\"🎯 Ensemble HitRate@3: {ensemble_hr3:.4f}\")\nexcept:\n    print(\"⚠️ HitRate function not available\")\n\n# Create submission\nsubmission_df = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', ensemble_test_preds))\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n)\n\ntry:\n    def re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, penalty_factor=0.12):\n        COLS_TO_COMPARE = [\n            \"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\",\n            \"legs0_segments0_flightNumber\", \"legs1_segments0_flightNumber\",\n            \"legs0_segments0_aircraft_code\", \"legs1_segments0_aircraft_code\",\n            \"legs0_segments0_departureFrom_airport_iata\", \"legs1_segments0_departureFrom_airport_iata\",\n        ]\n        test = test.with_columns([pl.col(c).cast(str).fill_null(\"NULL\") for c in COLS_TO_COMPARE])\n        df = submission_xgb.join(test, on=[\"Id\", \"ranker_id\"], how=\"left\")\n        df = df.with_columns(\n            (pl.col(\"legs0_departureAt\") + \"_\" + pl.col(\"legs0_arrivalAt\") + \"_\" + \n             pl.col(\"legs1_departureAt\") + \"_\" + pl.col(\"legs1_arrivalAt\") + \"_\" +\n             pl.col(\"legs0_segments0_flightNumber\") + \"_\" + pl.col(\"legs1_segments0_flightNumber\")\n            ).alias(\"flight_hash\")\n        )\n        df = df.with_columns(\n            pl.max(\"pred_score\").over([\"ranker_id\", \"flight_hash\"]).alias(\"max_score_same_flight\")\n        )\n        df = df.with_columns(\n            (pl.col(\"pred_score\") - penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\"))\n            ).alias(\"reorder_score\")\n        )\n        df = df.with_columns(\n            pl.col(\"reorder_score\").rank(method=\"ordinal\", descending=True)\n            .over(\"ranker_id\").cast(pl.Int32).alias(\"new_selected\")\n        )\n        return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])\n\n    top = re_rank(test, submission_df)\n    submission_df = (\n        submission_df.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n        .with_columns([\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ])\n        .select([\"Id\", \"ranker_id\", \"selected\"])\n    )\n    print(\"✅ Re-ranking applied\")\nexcept:\n    submission_df = submission_df.select([\"Id\", \"ranker_id\", \"selected\"])\n    print(\"⚠️ Re-ranking skipped\")\n\n# Save submission\nsubmission_df.write_csv('submission.csv')\nprint(f\"Final Validation Score: {best_score:.4f}\")\nprint(f\"Submission saved as 'submission.csv'\")\nprint(f\"Total models in ensemble: {len(all_models)}\")\nprint(submission_df.head())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}