{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U xgboost\n!pip install -U polars\n!pip install -U optuna\n!pip install -U catboost\n!pip install -U lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:59:11.405301Z","iopub.execute_input":"2025-08-14T14:59:11.405592Z","iopub.status.idle":"2025-08-14T14:59:53.978829Z","shell.execute_reply.started":"2025-08-14T14:59:11.405564Z","shell.execute_reply":"2025-08-14T14:59:53.973994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:59:54.442675Z","iopub.execute_input":"2025-08-14T14:59:54.442959Z","iopub.status.idle":"2025-08-14T14:59:54.461310Z","shell.execute_reply.started":"2025-08-14T14:59:54.442922Z","shell.execute_reply":"2025-08-14T14:59:54.455578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nimport catboost\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import ndcg_score\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:59:54.463141Z","iopub.execute_input":"2025-08-14T14:59:54.463344Z","iopub.status.idle":"2025-08-14T14:59:59.954532Z","shell.execute_reply.started":"2025-08-14T14:59:54.463324Z","shell.execute_reply":"2025-08-14T14:59:59.949158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:59:59.955334Z","iopub.execute_input":"2025-08-14T14:59:59.955647Z","iopub.status.idle":"2025-08-14T15:00:04.237437Z","shell.execute_reply.started":"2025-08-14T14:59:59.955624Z","shell.execute_reply":"2025-08-14T15:00:04.232321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_hubs = (\n    train.select(\"legs0_segments1_departureFrom_airport_iata\")\n         .drop_nulls()\n         .group_by(\"legs0_segments1_departureFrom_airport_iata\")\n         .agg(pl.count().alias(\"count\"))\n         .sort(\"count\", descending=True)\n         .head(10)\n)\n\nhub_airports = top_hubs[\"legs0_segments1_departureFrom_airport_iata\"].to_list()\nhub_airports","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:00:04.239539Z","iopub.execute_input":"2025-08-14T15:00:04.239804Z","iopub.status.idle":"2025-08-14T15:00:04.415705Z","shell.execute_reply.started":"2025-08-14T15:00:04.239780Z","shell.execute_reply":"2025-08-14T15:00:04.410458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:00:04.417551Z","iopub.execute_input":"2025-08-14T15:00:04.417790Z","iopub.status.idle":"2025-08-14T15:00:04.428832Z","shell.execute_reply.started":"2025-08-14T15:00:04.417768Z","shell.execute_reply":"2025-08-14T15:00:04.423299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\n\n# Combine all initial transformations\ndf = df.with_columns([\n        # Price features\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # Duration features\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n        \n        # Trip type\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # Total segments count\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # FF features\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # Binary features\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n        # Baggage & fees\n        # (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n        #  pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n        # (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + \n        #  pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n\n        (\n            (pl.col(\"miniRules0_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules0_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_cancel\"),\n        (\n            (pl.col(\"miniRules1_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules1_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_exchange\"),\n    \n        # Routes & carriers\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # Cabin\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n])\n\n# Segment counts - more efficient\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n    else:\n        seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n# Add segment-based features\n# First create segment counts\ndf = df.with_columns(seg_exprs)\n\n# Then use them for derived features\ndf = df.with_columns([\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n])\n\n# More derived features\ndf = df.with_columns([\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    # (pl.col(\"baggage_total\") > 0).cast(pl.Int32).alias(\"has_baggage\"),\n    # (pl.col(\"total_fees\") > 0).cast(pl.Int32).alias(\"has_fees\"),\n    # (pl.col(\"total_fees\") / (pl.col(\"totalPrice\") + 1)).alias(\"fee_rate\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n])\n\n# Add major carrier flag if column exists\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns(\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\")\n    )\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\ndf = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n# Time features - batch process\ntime_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# Batch rank computations - more efficient with single pass\n# First apply the columns that will be used for ranking\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n])\n\n# Price and duration basic ranks\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n# Price-specific features\nprice_exprs = [\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n]\n\n# Apply initial ranks\ndf = df.with_columns(rank_exprs + price_exprs)\n\n# Cheapest direct - more efficient\ndirect_cheapest = (\n    df.filter(pl.col(\"is_direct_leg0\") == 1)\n    .group_by(\"ranker_id\")\n    .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n)\n\ndf = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n    ((pl.col(\"is_direct_leg0\") == 1) & \n     (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n).drop(\"min_direct\")\n\n# Popularity features - efficient join\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n    ])\n)\n\n# Step 1: Add independent features\ndf = df.with_columns([\n    # Carrier popularity\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n])\n\ndf = df.with_columns(\n    (\n        # Policy compliance (25% weight)\n        (pl.col(\"pricingInfo_isAccessTP\") * 0.25) +\n        # Direct flights (25% weight)\n        (pl.col(\"is_direct_leg0\") * 0.25) +\n        # Business-hour departures/arrivals (25% weight)\n        ((pl.col(\"legs0_departureAt_business_time\") + pl.col(\"legs1_departureAt_business_time\")) * 0.125) +\n        # VIP preference for business class (25% weight)\n        ((pl.col(\"isVip\") == 1) & (pl.col(\"avg_cabin_class\") >= 1.5)).cast(pl.Int8) * 0.25\n    ).alias(\"business_traveler_perfect_match\"),\n\n    # Timezone diff only\n    (pl.col(\"legs0_arrivalAt_hour\") - pl.col(\"legs0_departureAt_hour\") -\n     (pl.col(\"legs0_duration\") / 60)).alias(\"timezone_diff_leg0\"),\n)\n\ndf = df.with_columns(\n    (\n        (pl.col(\"is_one_way\") == 0) &  # Round-trip\n        (pl.col(\"legs0_arrivalAt_hour\") >= 8) &  # Arrive by morning\n        (pl.col(\"legs1_departureAt_hour\") <= 18) &  # Return by evening\n        (pl.col(\"timezone_diff_leg0\").abs() < 3)   # Minimal jetlag\n    ).cast(pl.Int8).alias(\"meeting_friendly_itinerary\")\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:00:04.431362Z","iopub.execute_input":"2025-08-14T15:00:04.432010Z","iopub.status.idle":"2025-08-14T15:01:19.592664Z","shell.execute_reply.started":"2025-08-14T15:00:04.431985Z","shell.execute_reply":"2025-08-14T15:01:19.588808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = df.with_columns(\n    [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n    [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:19.596078Z","iopub.execute_input":"2025-08-14T15:01:19.596382Z","iopub.status.idle":"2025-08-14T15:01:23.480245Z","shell.execute_reply.started":"2025-08-14T15:01:19.596354Z","shell.execute_reply":"2025-08-14T15:01:23.475319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#df = df.with_columns([\n    #(df[\"legs0_segments0_departureFrom_airport_iata\"] == df[\"legs1_segments0_arrivalTo_airport_iata\"]).cast(pl.Int8).alias(\"roundtrip_symmetric\"),\n    #(df[\"carrier0_pop\"] - df[\"carrier1_pop\"]).fill_null(0).alias(\"carrier_popularity_diff\"),\n    #(1 / (df[\"group_size\"] + 1)).alias(\"inv_group_size\"),\n    # (pl.col(\"is_min_segments\") == True) & (pl.col(\"group_size\") <= 15).cast(pl.Int32).fill_null(0).alias(\"is_min_segmentand_group_size\")\n#])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:23.482464Z","iopub.execute_input":"2025-08-14T15:01:23.482733Z","iopub.status.idle":"2025-08-14T15:01:23.491818Z","shell.execute_reply.started":"2025-08-14T15:01:23.482693Z","shell.execute_reply":"2025-08-14T15:01:23.487399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber'\n]\n\n# Columns to exclude (uninformative or problematic)\nexclude_cols = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n    'frequentFlyer',  # Already processed\n    # Exclude constant columns\n    'pricingInfo_passengerCount','bySelf','n_segments_leg1','timezone_diff_leg0','meeting_friendly_itinerary','business_traveler_perfect_match'\n]\n\nfor leg in [0, 1]:\n    for seg in [0, 1]:\n        if seg == 0:\n            suffixes = [\n                \"seatsAvailable\",\n            ]\n        else:\n            suffixes = [\n                \"cabinClass\",\n                \"seatsAvailable\",\n                \"baggageAllowance_quantity\",\n                \"baggageAllowance_weightMeasurementType\",\n                \"aircraft_code\",\n                \"arrivalTo_airport_city_iata\",\n                \"arrivalTo_airport_iata\",\n                \"departureFrom_airport_iata\",\n                \"flightNumber\",\n                \"marketingCarrier_code\",\n                \"operatingCarrier_code\",\n            ]\n        for suffix in suffixes:\n            exclude_cols.append(f\"legs{leg}_segments{seg}_{suffix}\")\n\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols + ['ranker_id'])\ny = data.select(['selected', 'ranker_id'])\ngroups = data.select('ranker_id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:23.494609Z","iopub.execute_input":"2025-08-14T15:01:23.494832Z","iopub.status.idle":"2025-08-14T15:01:23.515536Z","shell.execute_reply.started":"2025-08-14T15:01:23.494811Z","shell.execute_reply":"2025-08-14T15:01:23.510988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb = X.with_columns([(pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int16) for c in cat_features_final])\n\nn1 = 16487352 # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n2], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n2], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n2], groups[n1:n2], groups[n2:]\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\ngroup_sizes_tr_lgb = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_va_lgb = groups_va.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_te_lgb = groups_te.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:23.518283Z","iopub.execute_input":"2025-08-14T15:01:23.518520Z","iopub.status.idle":"2025-08-14T15:01:29.348907Z","shell.execute_reply.started":"2025-08-14T15:01:23.518501Z","shell.execute_reply":"2025-08-14T15:01:29.345307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Step 1: group size & median ---\nranker_group_sizes = groups_tr.group_by('ranker_id').agg(pl.len().alias('group_size'))\nmedian_group_size = ranker_group_sizes['group_size'].median()\nprint(f\"Median group size: {median_group_size}\")\n\n# --- Step 2: get ranker_id lists ---\nsmall_rankers = ranker_group_sizes.filter(pl.col('group_size') < median_group_size)['ranker_id']\nbig_rankers   = ranker_group_sizes.filter(pl.col('group_size') >= median_group_size)['ranker_id']\n\n# --- Step 3: rankers where is_min_segments == 0 ---\nmin_segment_0 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 0)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nmin_segment_1 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nis_one_way_ranker = (\n    data_xgb_tr\n    .filter(pl.col('is_one_way') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nis_popular_route_ranker = (\n    data_xgb_tr\n    .filter(pl.col('is_popular_route') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\n# --- Step 4: filtering helper ---\ndef split_data(ranker_ids):\n    ranker_list = ranker_ids.to_list() if hasattr(ranker_ids, \"to_list\") else list(ranker_ids)\n    data = data_xgb_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    y    = y_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    grp  = groups_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    return (\n        data.drop('ranker_id').to_pandas(),\n        y.drop('ranker_id')['selected'].to_numpy(),\n        grp\n    )\n\n\n# last_minute_rankers = (\n#     data_xgb_tr\n#     .with_columns(\n#         (pl.col(\"legs0_departureAt\").str.to_datetime() - pl.col(\"search_date\").str.to_datetime())\n#         .dt.days().alias(\"days_until_departure\")\n#     )\n#     .filter(pl.col(\"days_until_departure\") <= 3)\n#     .select(\"ranker_id\")\n#     .unique()[\"ranker_id\"]\n# )\n\n# 2. Cheapest flights in each search\ncheapest_rankers = (\n    data_xgb_tr\n    .filter(pl.col(\"is_cheapest\") == 1)\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 3. Direct flights only\ndirect_flight_rankers = (\n    data_xgb_tr\n    .filter((pl.col(\"is_direct_leg0\") == 1) & (pl.col(\"is_one_way\") == 1))\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 4. High loyalty passengers\nloyalty_rankers = (\n    data_xgb_tr\n    .filter(pl.col(\"n_ff_programs\") >= 2)\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 5. Business-friendly itineraries\n# biz_friendly_rankers = (\n#     data_xgb_tr\n#     .filter(pl.col(\"business_traveler_perfect_match\") >= 0.75)\n#     .select(\"ranker_id\")\n#     .unique()[\"ranker_id\"]\n# )\n\n# --- Step Z: create new scenario datasets ---\ndata_xgb_tr_small_pd, y_tr_small_pd, groups_tr_small   = split_data(small_rankers)\ndata_xgb_tr_big_pd,   y_tr_big_pd,   groups_tr_big     = split_data(big_rankers)\ndata_xgb_tr_min0_pd,  y_tr_min0_pd,  groups_tr_min0    = split_data(min_segment_0)\ndata_xgb_tr_one_way_pd,  y_tr_one_way_pd,  groups_tr_one_way    = split_data(is_one_way_ranker)\ndata_xgb_tr_popular_route_pd,  y_tr_popular_route_pd,  groups_tr_popular_route_way    = split_data(is_popular_route_ranker)\n#data_xgb_tr_last_minute_pd, y_tr_last_minute_pd, groups_tr_last_minute = split_data(last_minute_rankers)\n#data_xgb_tr_cheapest_pd,   y_tr_cheapest_pd,   groups_tr_cheapest     = split_data(cheapest_rankers)\n#data_xgb_tr_direct_pd,     y_tr_direct_pd,     groups_tr_direct       = split_data(direct_flight_rankers)\ndata_xgb_tr_loyalty_pd,    y_tr_loyalty_pd,    groups_tr_loyalty      = split_data(loyalty_rankers)\n#data_xgb_tr_biz_pd,        y_tr_biz_pd,        groups_tr_biz          = split_data(biz_friendly_rankers)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:29.351964Z","iopub.execute_input":"2025-08-14T15:01:29.352190Z","iopub.status.idle":"2025-08-14T15:01:39.114676Z","shell.execute_reply.started":"2025-08-14T15:01:29.352169Z","shell.execute_reply":"2025-08-14T15:01:39.109977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb_tr.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:39.116623Z","iopub.execute_input":"2025-08-14T15:01:39.116923Z","iopub.status.idle":"2025-08-14T15:01:39.128415Z","shell.execute_reply.started":"2025-08-14T15:01:39.116898Z","shell.execute_reply":"2025-08-14T15:01:39.124911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"helloooo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:01:39.131903Z","iopub.execute_input":"2025-08-14T15:01:39.133546Z","iopub.status.idle":"2025-08-14T15:01:39.143082Z","shell.execute_reply.started":"2025-08-14T15:01:39.133520Z","shell.execute_reply":"2025-08-14T15:01:39.137333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(min_segment_1) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:11:54.152924Z","iopub.execute_input":"2025-08-14T15:11:54.153234Z","iopub.status.idle":"2025-08-14T15:11:54.166516Z","shell.execute_reply.started":"2025-08-14T15:11:54.153208Z","shell.execute_reply":"2025-08-14T15:11:54.161391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import ndcg_score\n\n# ----------------- Constants and Globals -----------------\nRANDOM_STATE = 42\nshap_lgbm_seed = {}\nxgb_seeds = [50]\nlgb_seeds = [42]\nall_models = []\nfeature_importances = []\n\n# ----------------- XGBoost Params -----------------\nxgb_rank_params = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_small = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 22,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_big = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_min0 = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_params = {\n    'full':    xgb_rank_params,\n    'small':   xgb_rank_params,\n    'big':     xgb_rank_params,\n    'min0':    xgb_rank_params,\n    #'min1':    xgb_rank_params,\n    'one_way': xgb_rank_params,\n    #'last_minute': xgb_rank_params,\n    #'cheapest':    xgb_rank_params,\n    'loyalty':     xgb_rank_params,\n    'popular_route': xgb_rank_params,\n    #'biz':         xgb_rank_params\n}\n\n# ----------------- LightGBM Params -----------------\nlgb_rank_params = {\n    'objective': 'lambdarank',\n    'metric': 'ndcg',\n    'boosting_type': 'gbdt',\n    'eval_at': [3],\n    'num_leaves': 137,\n    'learning_rate': 0.1923609,\n    'min_child_samples': 69,\n    'lambda_l1': 0.0017863,\n    'lambda_l2': 7.8818,\n    'feature_fraction': 0.6015,\n    'bagging_fraction': 0.8536,\n    'bagging_freq': 7,\n    'verbosity': -1,\n    'label_gain': [0, 1]\n}\n\n# ----------------- Drop ranker_id from pandas frames -----------------\nfor df in [\n    data_xgb_tr_small_pd, data_xgb_tr_big_pd, data_xgb_tr_min0_pd,\n    data_xgb_tr_one_way_pd, data_xgb_tr_popular_route_pd,\n    data_xgb_tr_loyalty_pd\n]:\n    if 'ranker_id' in df.columns:\n        df.drop(columns=['ranker_id'], inplace=True)\n\n# ----------------- Group sizes for small/big -----------------\ngroup_sizes_tr_small        = groups_tr_small.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_big          = groups_tr_big.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_min_0         = groups_tr_min0.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_size_tr_min_1        = groups_tr_min1.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_one_way        = groups_tr_one_way.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_popular_route        = groups_tr_popular_route_way.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_last_minute  = groups_tr_last_minute.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_cheapest     = groups_tr_cheapest.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_direct       = groups_tr_direct.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_loyalty      = groups_tr_loyalty.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_biz          = groups_tr_biz.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\n# ----------------- DMatrix Setup -----------------\ndtrain_full = xgb.DMatrix(\n    data_xgb_tr.drop('ranker_id').to_pandas(),\n    label=y_tr['selected'].to_numpy(),\n    group=group_sizes_tr,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtrain_small = xgb.DMatrix(data_xgb_tr_small_pd, label=y_tr_small_pd, group=group_sizes_tr_small, feature_names=list(data_xgb_tr_small_pd.columns))\ndtrain_big   = xgb.DMatrix(data_xgb_tr_big_pd,   label=y_tr_big_pd,   group=group_sizes_tr_big,   feature_names=list(data_xgb_tr_big_pd.columns))\ndtrain_min_0 = xgb.DMatrix(data_xgb_tr_min0_pd,  label=y_tr_min0_pd,  group=group_size_tr_min_0,  feature_names=list(data_xgb_tr_min0_pd.columns))\n#dtrain_min_1 = xgb.DMatrix(data_xgb_tr_min1_pd,  label=y_tr_min1_pd,  group=group_size_tr_min_1,  feature_names=list(data_xgb_tr_min1_pd.columns))\ndtrain_one_way = xgb.DMatrix(data_xgb_tr_one_way_pd,  label=y_tr_one_way_pd,  group=group_size_tr_one_way,  feature_names=list(data_xgb_tr_one_way_pd.columns))\ndtrain_popular_route = xgb.DMatrix(data_xgb_tr_popular_route_pd,  label=y_tr_popular_route_pd,  group=group_size_tr_popular_route,  feature_names=list(data_xgb_tr_popular_route_pd.columns))\n\n# New scenarios\n#dtrain_last_minute = xgb.DMatrix(data_xgb_tr_last_minute_pd, label=y_tr_last_minute_pd, group=group_sizes_tr_last_minute, feature_names=list(data_xgb_tr_last_minute_pd.columns))\n#dtrain_cheapest    = xgb.DMatrix(data_xgb_tr_cheapest_pd,    label=y_tr_cheapest_pd,    group=group_sizes_tr_cheapest,    feature_names=list(data_xgb_tr_cheapest_pd.columns))\ndtrain_loyalty     = xgb.DMatrix(data_xgb_tr_loyalty_pd,     label=y_tr_loyalty_pd,     group=group_sizes_tr_loyalty,     feature_names=list(data_xgb_tr_loyalty_pd.columns))\n#dtrain_biz         = xgb.DMatrix(data_xgb_tr_biz_pd,         label=y_tr_biz_pd,         group=group_sizes_tr_biz,         feature_names=list(data_xgb_tr_biz_pd.columns))\n\n# Validation and Test remain unchanged\ndval = xgb.DMatrix(data_xgb_va.drop('ranker_id').to_pandas(), label=y_va['selected'].to_numpy(), group=group_sizes_va, feature_names=data_xgb.drop('ranker_id').columns)\ndtest = xgb.DMatrix(data_xgb_te.drop('ranker_id').to_pandas(), label=y_te['selected'].to_numpy(), group=group_sizes_te, feature_names=data_xgb.drop('ranker_id').columns)\n\n# ----------------- Train XGBoost Models -----------------\nfor portion, params in xgb_params.items():\n    print(f\"\\nTraining XGBoost model with portion: {portion}...\")\n\n    train_map = {\n        'full':        dtrain_full,\n        'small':       dtrain_small,\n        'big':         dtrain_big,\n        'min0':        dtrain_min_0,\n        #'min1':        dtrain_min_1,\n        'one_way':     dtrain_one_way,\n        'popular_route': dtrain_popular_route,\n        #'last_minute': dtrain_last_minute,\n        #'cheapest':    dtrain_cheapest,\n        #'direct':      dtrain_direct,\n        'loyalty':     dtrain_loyalty,\n        #'biz':         dtrain_biz\n    }\n\n    train = train_map[portion]\n    val   = dval\n\n    params = params.copy()\n    params['seed'] = RANDOM_STATE\n\n    model = xgb.train(\n        params,\n        train,\n        num_boost_round=860,\n        evals=[(train, 'train'), (val, 'val')],\n        # early_stopping_rounds=100,\n        verbose_eval=50\n    )\n\n    all_models.append(('xgb', RANDOM_STATE, portion, model))\n\n    # Feature importance\n    xgb_fi = pd.DataFrame.from_dict(\n        model.get_score(importance_type='gain'),\n        orient='index',\n        columns=['importance_gain']\n    ).reset_index()\n    xgb_fi.columns = ['feature', 'importance_gain']\n    xgb_fi['importance_split'] = list(model.get_score(importance_type='weight').values())\n    xgb_fi['seed'] = RANDOM_STATE\n    xgb_fi['model_type'] = 'xgb'\n    xgb_fi['portion'] = portion\n    feature_importances.append(xgb_fi)\n\n# ----------------- Optional: Train LightGBM -----------------\n# Uncomment if needed\n\n# for seed in lgb_seeds:\n#     print(f\"\\nTraining LightGBM model with seed {seed}...\")\n#     lgb_train = lgb.Dataset(\n#         data=data_xgb_tr.drop('ranker_id').to_pandas(),\n#         label=y_tr['selected'].to_numpy(),\n#         group=group_sizes_tr_lgb,\n#         feature_name=data_xgb.drop('ranker_id').columns,\n#         free_raw_data=False\n#     )\n#     lgb_val = lgb.Dataset(\n#         data=data_xgb_va.drop('ranker_id').to_pandas(),\n#         label=y_va['selected'].to_numpy(),\n#         group=group_sizes_va_lgb,\n#         feature_name=data_xgb.drop('ranker_id').columns,\n#         reference=lgb_train,\n#         free_raw_data=False\n#     )\n#     params = lgb_rank_params.copy()\n#     params['seed'] = seed\n\n#     model = lgb.train(\n#         params,\n#         lgb_train,\n#         num_boost_round=1700,\n#         valid_sets=[lgb_train, lgb_val],\n#         callbacks=[lgb.early_stopping(300), lgb.log_evaluation(50)]\n#     )\n\n#     all_models.append(('lgb', seed, model))\n\n#     fi_df = pd.DataFrame({\n#         'feature': data_xgb.drop('ranker_id').columns,\n#         'importance_split': model.feature_importance(importance_type='split'),\n#         'importance_gain': model.feature_importance(importance_type='gain'),\n#         'seed': seed,\n#         'model_type': 'lgb'\n#     })\n#     feature_importances.append(fi_df)\n\n# ----------------- Combine Feature Importances -----------------\nall_feature_importance = pd.concat(feature_importances, ignore_index=True)\n\nprint(\"\\n✅ Training completed. Total models:\", len(all_models))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:16:44.019054Z","iopub.execute_input":"2025-08-14T15:16:44.019403Z","iopub.status.idle":"2025-08-14T15:18:18.744312Z","shell.execute_reply.started":"2025-08-14T15:16:44.019374Z","shell.execute_reply":"2025-08-14T15:18:18.739342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by feature and model_type\nagg_importance = (\n    all_feature_importance\n    .groupby([\"model_type\", \"feature\"])[[\"importance_split\", \"importance_gain\"]]\n    .mean()\n    .reset_index()\n)\n\n# Separate and sort LightGBM and XGBoost\n# lgb_fi_sorted = (\n#     agg_importance[agg_importance[\"model_type\"] == \"lgb\"]\n#     .sort_values(\"importance_gain\", ascending=False)\n# )\n\nxgb_fi_sorted = (\n    agg_importance[agg_importance[\"model_type\"] == \"xgb\"]\n    .sort_values(\"importance_gain\", ascending=False)\n)\n\n# # Display top features\n# print(\"🔝 Top LightGBM Features (by Gain):\")\n# print(lgb_fi_sorted.head(30))\n\npd.set_option('display.max_rows', None)     # Show all rows\npd.set_option('display.max_columns', None)  # Show all columns\npd.set_option('display.width', 0)           # Auto-detect width\npd.set_option('display.max_colwidth', None) # Show full content in cells\n\nprint(xgb_fi_sorted.head(100))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:30:41.657334Z","iopub.execute_input":"2025-08-14T15:30:41.657675Z","iopub.status.idle":"2025-08-14T15:30:41.689162Z","shell.execute_reply.started":"2025-08-14T15:30:41.657648Z","shell.execute_reply":"2025-08-14T15:30:41.682806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import ndcg_score\nimport numpy as np\n\n# --- Separate LGB and XGB models from all_models ---\n#lgb_models = [m for (typ, _, m) in all_models if typ == 'lgb']\nxgb_models = [model for (typ, _, _, model) in all_models if typ == 'xgb']\n\n# --- Predict on validation set and average ---\nprint(\"\\n📊 Averaging predictions on validation set...\")\n\n# LightGBM predictions\n#preds_val_lgb = np.mean([model.predict(data_xgb_va) for model in lgb_models], axis=0)\n\n# XGBoost predictions\ndata_xgb_va_pd = data_xgb_va.drop('ranker_id').to_pandas()\ndval = xgb.DMatrix(data_xgb_va_pd)\npreds_val_xgb = np.mean([model.predict(dval) for model in xgb_models], axis=0)\n\n# Combine XGBoost predictions\npreds_val = preds_val_xgb\n\n# Convert labels to numeric if needed\ny_true = y_va['selected'].to_numpy().flatten()\n\n# Evaluate NDCG@3\nndcg_val = ndcg_score([y_true], [preds_val], k=3)\nprint(f\"✅ Ensemble NDCG@3: {ndcg_val:.4f}\")\n\n# Evaluate HitRate@3 (if function is defined)\nensemble_hr3 = hitrate_at_3(y_true, preds_val, groups_va['ranker_id'])\nprint(f\"🎯 Ensemble HitRate@3: {ensemble_hr3:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:30:45.856094Z","iopub.execute_input":"2025-08-14T15:30:45.856415Z","iopub.status.idle":"2025-08-14T15:30:47.857246Z","shell.execute_reply.started":"2025-08-14T15:30:45.856388Z","shell.execute_reply":"2025-08-14T15:30:47.850950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, penalty_factor=0.12):\n    COLS_TO_COMPARE = [\n        \"legs0_departureAt\",\n        \"legs0_arrivalAt\",\n        \"legs1_departureAt\",\n        \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\",\n        \"legs1_segments0_flightNumber\",\n        \"legs0_segments0_aircraft_code\",\n        \"legs1_segments0_aircraft_code\",\n        \"legs0_segments0_departureFrom_airport_iata\",\n        \"legs1_segments0_departureFrom_airport_iata\",\n    ]\n\n    test = test.with_columns(\n        [pl.col(c).cast(str).fill_null(\"NULL\") for c in COLS_TO_COMPARE]\n    )\n\n    df = submission_xgb.join(test, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    df = df.with_columns(\n        (\n            pl.col(\"legs0_departureAt\")\n            + \"_\"\n            + pl.col(\"legs0_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs1_departureAt\")\n            + \"_\"\n            + pl.col(\"legs1_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs0_segments0_flightNumber\")\n            + \"_\"\n            + pl.col(\"legs1_segments0_flightNumber\")\n        ).alias(\"flight_hash\")\n    )\n\n    df = df.with_columns(\n        pl.max(\"pred_score\")\n        .over([\"ranker_id\", \"flight_hash\"])\n        .alias(\"max_score_same_flight\")\n    )\n\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\")\n            - penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\"))\n        ).alias(\"reorder_score\")\n    )\n\n    df = df.with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:30:49.920440Z","iopub.execute_input":"2025-08-14T15:30:49.920783Z","iopub.status.idle":"2025-08-14T15:30:49.937767Z","shell.execute_reply.started":"2025-08-14T15:30:49.920757Z","shell.execute_reply":"2025-08-14T15:30:49.931777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Predict on test set ---\nprint(\"\\n📦 Generating predictions on test set...\")\n\n# LightGBM test predictions\n#preds_test_lgb = np.mean([model.predict(data_xgb_te) for model in lgb_models], axis=0)\n\n# # XGBoost test predictions\n\ndata_xgb_te_pd = data_xgb_te.drop('ranker_id').to_pandas()\ndtest = xgb.DMatrix(data_xgb_te_pd)\npreds_test_xgb = np.mean([model.predict(dtest) for model in xgb_models], axis=0)\n\n# Combined test ensemble\nensemble_test_preds =  preds_test_xgb\n\nsubmission_df = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', ensemble_test_preds))\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n)\n\ntop = re_rank(test, submission_df)\n\nsubmission_df = (\n    submission_df.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n    .with_columns(\n        [\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ]\n    )\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\n\n# --- Save to CSV ---\nsubmission_df.write_csv('submission.csv')\nprint(\"\\n✅ Submission file 'submission.csv' created successfully.\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:30:52.415876Z","iopub.execute_input":"2025-08-14T15:30:52.416248Z","iopub.status.idle":"2025-08-14T15:31:02.570948Z","shell.execute_reply.started":"2025-08-14T15:30:52.416222Z","shell.execute_reply":"2025-08-14T15:31:02.564247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import xgboost as xgb\n\n# xgb_rank_params = {\n#     'objective': 'rank:ndcg',       # or use 'rank:pairwise' as alternative\n#     'learning_rate': 0.1,\n#     'gamma': 1.0,\n#     'min_child_weight': 30,\n#     'max_depth': 6,\n#     'subsample': 0.85,\n#     'colsample_bytree': 0.6,\n#     'lambda': 1.0,\n#     'alpha': 0.1,\n#     'eval_metric': 'ndcg@3',\n#     'verbosity': 1,\n#     'seed': 42,\n#     'tree_method': 'hist',          # optional for speed\n# }\n\n# print(\"Training XGBoost ranker...\")\n\n# # Create DMatrix for XGBoost\n# xgb_train = xgb.DMatrix(data_xgb_tr, label=y_tr.to_numpy().flatten())\n# xgb_val = xgb.DMatrix(data_xgb_va, label=y_va.to_numpy().flatten())\n\n# xgb_train.set_group(group_sizes_tr)\n# xgb_val.set_group(group_sizes_va)\n\n# # Train XGBoost ranking model\n# xgb_model = xgb.train(\n#     xgb_rank_params,\n#     dtrain=xgb_train,\n#     num_boost_round=500,\n#     evals=[(xgb_train, \"train\"), (xgb_val, \"valid\")],\n#     early_stopping_rounds=50,\n#     verbose_eval=50\n# )\n\n# xgb_fi = pd.DataFrame.from_dict(xgb_model.get_score(importance_type='gain'), orient='index', columns=['importance_gain']).reset_index()\n# xgb_fi.columns = ['feature', 'importance_gain']\n# xgb_fi = xgb_fi.sort_values('importance_gain', ascending=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.407635Z","iopub.execute_input":"2025-08-09T18:08:16.407857Z","iopub.status.idle":"2025-08-09T18:08:16.417342Z","shell.execute_reply.started":"2025-08-09T18:08:16.407834Z","shell.execute_reply":"2025-08-09T18:08:16.412700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# xgb_fi.head(30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.420293Z","iopub.execute_input":"2025-08-09T18:08:16.420645Z","iopub.status.idle":"2025-08-09T18:08:16.433558Z","shell.execute_reply.started":"2025-08-09T18:08:16.420617Z","shell.execute_reply":"2025-08-09T18:08:16.425486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# import numpy as np\n# from sklearn.metrics import ndcg_score\n\n# lgb_rank_params = {\n#     'objective': 'lambdarank',\n#     'metric': 'ndcg',\n#     'boosting_type': 'gbdt',\n#     'eval_at': [3],\n#     'num_leaves': 137,\n#     'learning_rate': 0.1923609,\n#     'min_child_samples': 69,\n#     'lambda_l1': 0.0017863,\n#     'lambda_l2': 7.8818,\n#     'feature_fraction': 0.6015,\n#     'bagging_fraction': 0.8536,\n#     'bagging_freq': 7,\n#     'verbosity': -1,\n#     'label_gain': [0, 1]\n# }\n\n\n# models = []\n# lgb_seeds = [12, 32, 42]\n\n# feature_importances = []\n\n# for seed in lgb_seeds:\n#     params = lgb_rank_params.copy()\n#     params['seed'] = seed\n\n#     print(f\"Training LightGBM model with seed {seed}...\")\n    \n#     lgb_train = lgb.Dataset(\n#         data=data_xgb_tr,\n#         label=y_tr.to_numpy().flatten(),\n#         group=group_sizes_tr,\n#         feature_name=feature_cols,\n#         free_raw_data=False\n#     )\n\n#     lgb_val = lgb.Dataset(\n#         data=data_xgb_va,\n#         label=y_va.to_numpy().flatten(),\n#         group=group_sizes_va,\n#         feature_name=feature_cols,\n#         reference=lgb_train,\n#         free_raw_data=False\n#     )\n\n#     model = lgb.train(\n#         params,\n#         lgb_train,\n#         num_boost_round=1500,\n#         valid_sets=[lgb_train, lgb_val],\n#         callbacks=[lgb.early_stopping(100), lgb.log_evaluation(50)]\n#     )\n\n#     models.append(model)\n\n#     # Store feature importance\n#     fi_df = pd.DataFrame({\n#         'feature': feature_cols,\n#         'importance_split': model.feature_importance(importance_type='split'),\n#         'importance_gain': model.feature_importance(importance_type='gain'),\n#         'seed': seed\n#     })\n#     feature_importances.append(fi_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.435170Z","iopub.execute_input":"2025-08-09T18:08:16.435381Z","iopub.status.idle":"2025-08-09T18:08:16.448032Z","shell.execute_reply.started":"2025-08-09T18:08:16.435355Z","shell.execute_reply":"2025-08-09T18:08:16.440199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Combine all importance dataframes\n# all_fi_df = pd.concat(feature_importances)\n\n# # Average importance across seeds\n# avg_fi = all_fi_df.groupby(\"feature\")[[\"importance_split\", \"importance_gain\"]].mean().sort_values(\"importance_gain\", ascending=False)\n\n# # Show top 20 features\n# print(avg_fi.head(30))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.448751Z","iopub.execute_input":"2025-08-09T18:08:16.448956Z","iopub.status.idle":"2025-08-09T18:08:16.458910Z","shell.execute_reply.started":"2025-08-09T18:08:16.448935Z","shell.execute_reply":"2025-08-09T18:08:16.453739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# avg_fi['norm_gain'] = 100 * avg_fi['importance_gain'] / avg_fi['importance_gain'].sum()\n# avg_fi['norm_gain'].head(20) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.459808Z","iopub.execute_input":"2025-08-09T18:08:16.460033Z","iopub.status.idle":"2025-08-09T18:08:16.469733Z","shell.execute_reply.started":"2025-08-09T18:08:16.460011Z","shell.execute_reply":"2025-08-09T18:08:16.464781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# avg_fi.head(30).sort_values(\"importance_gain\").plot(kind='barh', figsize=(15, 12))\n# plt.title(\"Top 30 Features by Gain\")\n# plt.xlabel(\"Average Gain Importance\")\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.472118Z","iopub.execute_input":"2025-08-09T18:08:16.472333Z","iopub.status.idle":"2025-08-09T18:08:16.482605Z","shell.execute_reply.started":"2025-08-09T18:08:16.472313Z","shell.execute_reply":"2025-08-09T18:08:16.475952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Predict on validation set and average\n# preds_val = np.mean([model.predict(data_xgb_va) for model in models], axis=0)\n\n# from sklearn.metrics import ndcg_score\n# # Evaluate using NDCG@3\n# ndcg_val = ndcg_score([y_va.to_numpy().flatten()], [preds_val], k=3)\n# print(f\"✅ Ensemble NDCG@3: {ndcg_val:.4f}\")\n\n# lgb_ensemble_hr3 = hitrate_at_3(y_va['selected'], preds_val, groups_va['ranker_id'])\n# print(f\"LGBM ensemble HitRate@3:   {lgb_ensemble_hr3:.4f}\")\n\n# lgb_ensemble_test_preds  = np.mean([model.predict(data_xgb_te) for model in models], axis=0)\n\n# submission_df = test.select(['Id', 'ranker_id']).with_columns(\n#     pl.Series(name=\"lgb_score\", values=lgb_ensemble_test_preds)\n# ).with_columns(\n#     # Rank predictions descending (best scores first) within each group\n#     pl.col(\"lgb_score\").rank(method=\"ordinal\", descending=True).over(\"ranker_id\").cast(pl.Int32).alias(\"selected\")\n# ).select([\"Id\", \"ranker_id\", \"selected\"])\n\n# # Save to CSV\n# submission_df.write_csv('submission.csv')\n\n# print(\"\\n✅ Submission file 'submission.csv' created successfully.\")\n# print(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T18:08:16.484136Z","iopub.execute_input":"2025-08-09T18:08:16.484334Z","iopub.status.idle":"2025-08-09T18:08:16.495386Z","shell.execute_reply.started":"2025-08-09T18:08:16.484315Z","shell.execute_reply":"2025-08-09T18:08:16.488771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}