{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars optuna -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:00.673556Z","iopub.execute_input":"2025-08-16T17:23:00.673923Z","iopub.status.idle":"2025-08-16T17:23:12.191286Z","shell.execute_reply.started":"2025-08-16T17:23:00.673891Z","shell.execute_reply":"2025-08-16T17:23:12.186690Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:12.192351Z","iopub.execute_input":"2025-08-16T17:23:12.192584Z","iopub.status.idle":"2025-08-16T17:23:12.204520Z","shell.execute_reply.started":"2025-08-16T17:23:12.192548Z","shell.execute_reply":"2025-08-16T17:23:12.200130Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nimport catboost\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import ndcg_score\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:12.206756Z","iopub.execute_input":"2025-08-16T17:23:12.207054Z","iopub.status.idle":"2025-08-16T17:23:12.221345Z","shell.execute_reply.started":"2025-08-16T17:23:12.207030Z","shell.execute_reply":"2025-08-16T17:23:12.217081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:12.222564Z","iopub.execute_input":"2025-08-16T17:23:12.222787Z","iopub.status.idle":"2025-08-16T17:23:15.980812Z","shell.execute_reply.started":"2025-08-16T17:23:12.222764Z","shell.execute_reply":"2025-08-16T17:23:15.976032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_hubs = (\n    train.select(\"legs0_segments1_departureFrom_airport_iata\")\n         .drop_nulls()\n         .group_by(\"legs0_segments1_departureFrom_airport_iata\")\n         .agg(pl.count().alias(\"count\"))\n         .sort(\"count\", descending=True)\n         .head(10)\n)\n\nhub_airports = top_hubs[\"legs0_segments1_departureFrom_airport_iata\"].to_list()\nhub_airports","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:15.982991Z","iopub.execute_input":"2025-08-16T17:23:15.983215Z","iopub.status.idle":"2025-08-16T17:23:16.124958Z","shell.execute_reply.started":"2025-08-16T17:23:15.983192Z","shell.execute_reply":"2025-08-16T17:23:16.119860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:23:16.127252Z","iopub.execute_input":"2025-08-16T17:23:16.127519Z","iopub.status.idle":"2025-08-16T17:23:16.139822Z","shell.execute_reply.started":"2025-08-16T17:23:16.127492Z","shell.execute_reply":"2025-08-16T17:23:16.133359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\n\n# Combine all initial transformations\ndf = df.with_columns([\n        # Price features\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # Duration features\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n        \n        # Trip type\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # Total segments count\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # FF features\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # Binary features\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n        # Baggage & fees\n        # (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n        #  pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n        # (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + \n        #  pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n\n        (\n            (pl.col(\"miniRules0_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules0_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_cancel\"),\n        (\n            (pl.col(\"miniRules1_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules1_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_exchange\"),\n    \n        # Routes & carriers\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # Cabin\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n])\n\n# Segment counts - more efficient\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n    else:\n        seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n# Add segment-based features\n# First create segment counts\ndf = df.with_columns(seg_exprs)\n\n# Then use them for derived features\ndf = df.with_columns([\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n])\n\n# More derived features\ndf = df.with_columns([\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n    \n    # New: Interactions\n    (pl.col(\"totalPrice\") / (pl.col(\"total_duration\") + 1)).alias(\"price_per_minute\"),  # Value metric\n    (pl.col(\"totalPrice\") * pl.col(\"avg_cabin_class\")).alias(\"price_cabin_interaction\"),  # Premium pricing\n    \n    # New: Aggregates over groups/carriers\n    pl.col(\"totalPrice\").mean().over(\"legs0_segments0_marketingCarrier_code\").alias(\"avg_price_per_carrier\"),\n    pl.col(\"selected\").mean().over(\"legs0_segments0_marketingCarrier_code\").alias(\"carrier_selection_rate\"),  # Historical preference\n])\n\n# Add major carrier flag if column exists\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns(\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\")\n    )\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\ndf = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n# Time features - batch process\ntime_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# Batch rank computations - more efficient with single pass\n# First apply the columns that will be used for ranking\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n])\n\n# Price and duration basic ranks\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n# Price-specific features\nprice_exprs = [\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n]\n\n# Apply initial ranks\ndf = df.with_columns(rank_exprs + price_exprs)\n\n# Cheapest direct - more efficient\ndirect_cheapest = (\n    df.filter(pl.col(\"is_direct_leg0\") == 1)\n    .group_by(\"ranker_id\")\n    .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n)\n\ndf = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n    ((pl.col(\"is_direct_leg0\") == 1) & \n     (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n).drop(\"min_direct\")\n\n# Popularity features - efficient join\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n    ])\n)\n\n# Step 1: Add independent features\ndf = df.with_columns([\n    # Carrier popularity\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n])\n\ndf = df.with_columns(\n    (\n        # Policy compliance (25% weight)\n        (pl.col(\"pricingInfo_isAccessTP\") * 0.25) +\n        # Direct flights (25% weight)\n        (pl.col(\"is_direct_leg0\") * 0.25) +\n        # Business-hour departures/arrivals (25% weight)\n        ((pl.col(\"legs0_departureAt_business_time\") + pl.col(\"legs1_departureAt_business_time\")) * 0.125) +\n        # VIP preference for business class (25% weight)\n        ((pl.col(\"isVip\") == 1) & (pl.col(\"avg_cabin_class\") >= 1.5)).cast(pl.Int8) * 0.25\n    ).alias(\"business_traveler_perfect_match\"),\n\n    # Timezone diff only\n    (pl.col(\"legs0_arrivalAt_hour\") - pl.col(\"legs0_departureAt_hour\") -\n     (pl.col(\"legs0_duration\") / 60)).alias(\"timezone_diff_leg0\"),\n)\n\ndf = df.with_columns(\n    (\n        (pl.col(\"is_one_way\") == 0) &  # Round-trip\n        (pl.col(\"legs0_arrivalAt_hour\") >= 8) &  # Arrive by morning\n        (pl.col(\"legs1_departureAt_hour\") <= 18) &  # Return by evening\n        (pl.col(\"timezone_diff_leg0\").abs() < 3)  # Minimal jetlag\n    ).cast(pl.Int8).alias(\"meeting_friendly_itinerary\")\n)\n\n# ================= NEW FEATURE BLOCK =================\n\n# --- Time-based Features ---\ndf = df.with_columns([\n    # Morning departure indicator\n    pl.col(\"legs0_departureAt_hour\")\n      .map_elements(lambda x: 1 if x is not None and 6 <= x <= 9 else 0)\n      .cast(pl.Int8)\n      .alias(\"is_morning_departure\"),\n\n    # Weekend flag (Friday=5, Saturday=6)\n    (pl.col(\"legs0_departureAt_weekday\") >= 5).cast(pl.Int8).alias(\"is_weekend_departure\"),\n\n    # Layover duration in minutes\n    (\n        (\n            pl.col(\"legs1_departureAt\").str.to_datetime(\"%Y-%m-%dT%H:%M:%S\", strict=False)\n            - pl.col(\"legs0_arrivalAt\").str.to_datetime(\"%Y-%m-%dT%H:%M:%S\", strict=False)\n        ).dt.total_seconds() / 60\n    ).fill_null(0).alias(\"layover_duration_mins\")\n])\n\n\n# --- Route-specific Features ---\ndf = df.with_columns([\n    pl.col(\"legs0_segments0_departureFrom_airport_iata\")\n      .is_in(hub_airports).cast(pl.Int8).alias(\"departs_from_hub\"),\n    pl.col(\"legs0_segments0_arrivalTo_airport_iata\")\n      .is_in(hub_airports).cast(pl.Int8).alias(\"arrives_at_hub\"),\n    (pl.col(\"legs0_segments0_departureFrom_airport_iata\") ==\n     pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).cast(pl.Int8).alias(\"symmetric_route\")\n])\n\n# --- User behavior / preference features ---\ndf = df.with_columns([\n    pl.col(\"selected\").mean().over(\"companyID\").fill_null(0).alias(\"company_selection_rate\"),\n    pl.col(\"totalPrice\").mean().over(\"companyID\").fill_null(0).alias(\"company_avg_price\"),\n    pl.col(\"selected\").mean().over(\"nationality\").fill_null(0).alias(\"nationality_preference\")\n])\n# ================= END NEW FEATURE BLOCK =================\n\n# --- Advanced Time-Based Travel Features ---\ndf = df.with_columns([\n    # Optimal time slots (business-preferred: 8–10 AM or 2–6 PM)\n    (((pl.col(\"legs0_departureAt_hour\").is_between(8,10)) |\n      (pl.col(\"legs0_departureAt_hour\").is_between(14,18)))\n     .cast(pl.Int8).alias(\"optimal_departure_time\")),\n\n    # Red-eye (overnight flights: 11 PM - 5 AM)\n    (((pl.col(\"legs0_departureAt_hour\") >= 23) | (pl.col(\"legs0_departureAt_hour\") <= 5))\n     .cast(pl.Int8).alias(\"is_redeye\")),\n\n    # Weekend premium (Friday=5, Sunday=0)\n    (((pl.col(\"legs0_departureAt_weekday\")==5) | (pl.col(\"legs0_departureAt_weekday\")==0))\n     .cast(pl.Int8).alias(\"weekend_premium\")),\n\n    # Duration efficiency — how close is itinerary to best option on same route\n    (pl.col(\"total_duration\") / \n     (pl.col(\"total_duration\").min().over(\"searchRoute\") + 1))\n    .alias(\"duration_efficiency\"),\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:32:11.503707Z","iopub.execute_input":"2025-08-16T17:32:11.504112Z","iopub.status.idle":"2025-08-16T17:33:58.188552Z","shell.execute_reply.started":"2025-08-16T17:32:11.504080Z","shell.execute_reply":"2025-08-16T17:33:58.183264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Advanced Price Intelligence ---\ndf = df.with_columns([\n    # Price ranks inside routes\n    pl.col(\"totalPrice\").rank().over(\"searchRoute\").alias(\"price_rank_route\"),\n    \n    # Route-specific quartiles\n    pl.col(\"totalPrice\").quantile(0.25).over(\"searchRoute\").alias(\"route_price_q25\"),\n    pl.col(\"totalPrice\").quantile(0.75).over(\"searchRoute\").alias(\"route_price_q75\"),\n\n    # Price anomaly score (z-score like detection inside route)\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"searchRoute\")).abs() / \n      (pl.col(\"totalPrice\").std().over(\"searchRoute\") + 1e-6))\n    .alias(\"price_anomaly_score\"),\n\n    # Value score = tradeoff price and duration relative to peer group\n    (1 / (pl.col(\"totalPrice\") / (pl.col(\"totalPrice\").median().over(\"ranker_id\")+1) +\n           pl.col(\"total_duration\") / (pl.col(\"total_duration\").median().over(\"ranker_id\")+1)))\n    .alias(\"value_score\"),\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:34:16.617213Z","iopub.execute_input":"2025-08-16T17:34:16.617531Z","iopub.status.idle":"2025-08-16T17:34:19.949779Z","shell.execute_reply.started":"2025-08-16T17:34:16.617505Z","shell.execute_reply":"2025-08-16T17:34:19.944486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = df.with_columns(\n    [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n    [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:37:36.813223Z","iopub.execute_input":"2025-08-16T17:37:36.813519Z","execution_failed":"2025-08-16T12:08:25.887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber'\n]\n\n# Columns to exclude (uninformative or problematic)\nexclude_cols = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n    'frequentFlyer',  # Already processed\n    # Exclude constant columns\n    'pricingInfo_passengerCount','bySelf','n_segments_leg1','timezone_diff_leg0','meeting_friendly_itinerary','business_traveler_perfect_match'\n]\n\nfor leg in [0, 1]:\n    for seg in [0, 1]:\n        if seg == 0:\n            suffixes = [\n                \"seatsAvailable\",\n            ]\n        else:\n            suffixes = [\n                \"cabinClass\",\n                \"seatsAvailable\",\n                \"baggageAllowance_quantity\",\n                \"baggageAllowance_weightMeasurementType\",\n                \"aircraft_code\",\n                \"arrivalTo_airport_city_iata\",\n                \"arrivalTo_airport_iata\",\n                \"departureFrom_airport_iata\",\n                \"flightNumber\",\n                \"marketingCarrier_code\",\n                \"operatingCarrier_code\",\n            ]\n        for suffix in suffixes:\n            exclude_cols.append(f\"legs{leg}_segments{seg}_{suffix}\")\n\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols + ['ranker_id'])\ny = data.select(['selected', 'ranker_id'])\ngroups = data.select('ranker_id')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-08-16T12:08:25.888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb = X.with_columns([(pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int16) for c in cat_features_final])\n\nn1 = 16487352 # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n2], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n2], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n2], groups[n1:n2], groups[n2:]\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\ngroup_sizes_tr_lgb = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_va_lgb = groups_va.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_te_lgb = groups_te.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-08-16T12:08:25.888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Step 1: group size & median ---\nranker_group_sizes = groups_tr.group_by('ranker_id').agg(pl.len().alias('group_size'))\nmedian_group_size = ranker_group_sizes['group_size'].median()\nprint(f\"Median group size: {median_group_size}\")\n\n# --- Step 2: get ranker_id lists ---\nsmall_rankers = ranker_group_sizes.filter(pl.col('group_size') < median_group_size)['ranker_id']\nbig_rankers   = ranker_group_sizes.filter(pl.col('group_size') >= median_group_size)['ranker_id']\n\n# --- Step 3: rankers where is_min_segments == 0 ---\nmin_segment_0 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 0)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\n\n\n# --- Step 4: filtering helper ---\ndef split_data(ranker_ids):\n    ranker_list = ranker_ids.to_list() if hasattr(ranker_ids, \"to_list\") else list(ranker_ids)\n    data = data_xgb_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    y    = y_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    grp  = groups_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    return (\n        data.drop('ranker_id').to_pandas(),\n        y.drop('ranker_id')['selected'].to_numpy(),\n        grp\n    )\n\n# --- Step 5: create splits ---\ndata_xgb_tr_small_pd, y_tr_small_pd, groups_tr_small   = split_data(small_rankers)\ndata_xgb_tr_big_pd,   y_tr_big_pd,   groups_tr_big     = split_data(big_rankers)\ndata_xgb_tr_min0_pd,  y_tr_min0_pd,  groups_tr_min0    = split_data(min_segment_0)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-08-16T12:08:25.888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_xgb_tr.shape","metadata":{"trusted":true,"execution":{"execution_failed":"2025-08-16T12:08:25.888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import ndcg_score\n\n# ----------------- Constants and Globals -----------------\nRANDOM_STATE = 42\nshap_lgbm_seed = {}\nxgb_seeds = [50]\nlgb_seeds = [42]\nall_models = []\nfeature_importances = []\n\n# ----------------- XGBoost Params -----------------\nxgb_rank_params = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_small = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 22,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_big = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_min0 = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_params = {'full': xgb_rank_params}","metadata":{"trusted":true,"execution":{"execution_failed":"2025-08-16T12:08:25.888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ----------------- Drop ranker_id from pandas frames -----------------\nfor df in [data_xgb_tr_small_pd, data_xgb_tr_big_pd]:\n    if 'ranker_id' in df.columns:\n        df.drop(columns=['ranker_id'], inplace=True)\n\n# ----------------- Group sizes for small/big -----------------\ngroup_sizes_tr_small = groups_tr_small.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_big = groups_tr_big.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_min_0 = groups_tr_min0.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\n# ----------------- DMatrix Setup -----------------\ndtrain_full = xgb.DMatrix(\n    data_xgb_tr.drop('ranker_id').to_pandas(),\n    label=y_tr['selected'].to_numpy(),\n    group=group_sizes_tr,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtrain_small = xgb.DMatrix(\n    data_xgb_tr_small_pd,\n    label=y_tr_small_pd,\n    group=group_sizes_tr_small,\n    feature_names=list(data_xgb_tr_small_pd.columns)\n)\n\ndtrain_big = xgb.DMatrix(\n    data_xgb_tr_big_pd,\n    label=y_tr_big_pd,\n    group=group_sizes_tr_big,\n    feature_names=list(data_xgb_tr_big_pd.columns)\n)\n\ndtrain_min_0 = xgb.DMatrix(\n    data_xgb_tr_min0_pd,\n    label=y_tr_min0_pd,\n    group=group_size_tr_min_0,\n    feature_names=list(data_xgb_tr_min0_pd.columns)\n)\n\ndval = xgb.DMatrix(\n    data_xgb_va.drop('ranker_id').to_pandas(),\n    label=y_va['selected'].to_numpy(),\n    group=group_sizes_va,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtest = xgb.DMatrix(\n    data_xgb_te.drop('ranker_id').to_pandas(),\n    label=y_te['selected'].to_numpy(),\n    group=group_sizes_te,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\n# ----------------- Train XGBoost Models -----------------\nfor type_of_portion in xgb_params:\n    print(f\"\\nTraining XGBoost model with portion: {type_of_portion}...\")\n\n    if type_of_portion == 'full':\n        train = dtrain_full\n        val = dval\n    elif type_of_portion == 'small':\n        train = dtrain_small\n        val = dval\n    elif type_of_portion == 'big':\n        train = dtrain_big\n        val = dval\n    elif type_of_portion == 'min0':\n        train = dtrain_min_0\n        val = dval\n\n    params = xgb_params[type_of_portion].copy()\n    params['seed'] = RANDOM_STATE\n\n    model = xgb.train(\n        params,\n        train,\n        num_boost_round=1800,\n        evals=[(train, 'train'), (val, 'val')],\n        # early_stopping_rounds=100,\n        verbose_eval=50\n    )\n\n    all_models.append(('xgb', RANDOM_STATE, type_of_portion, model))\n\n    # Feature importance\n    xgb_fi = pd.DataFrame.from_dict(\n        model.get_score(importance_type='gain'),\n        orient='index',\n        columns=['importance_gain']\n    ).reset_index()\n    xgb_fi.columns = ['feature', 'importance_gain']\n    xgb_fi['importance_split'] = list(model.get_score(importance_type='weight').values())\n    xgb_fi['seed'] = RANDOM_STATE\n    xgb_fi['model_type'] = 'xgb'\n    xgb_fi['portion'] = type_of_portion\n    feature_importances.append(xgb_fi)\n\n# ----------------- Combine Feature Importances -----------------\nall_feature_importance = pd.concat(feature_importances, ignore_index=True)\n\nprint(\"\\n✅ Training completed. Total models:\", len(all_models))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:24:24.340376Z","iopub.status.idle":"2025-08-16T17:24:24.340826Z","shell.execute_reply.started":"2025-08-16T17:24:24.340505Z","shell.execute_reply":"2025-08-16T17:24:24.340517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by feature and model_type\nagg_importance = (\n    all_feature_importance\n    .groupby([\"model_type\", \"feature\"])[[\"importance_split\", \"importance_gain\"]]\n    .mean()\n    .reset_index()\n)\n\nxgb_fi_sorted = (\n    agg_importance[agg_importance[\"model_type\"] == \"xgb\"]\n    .sort_values(\"importance_gain\", ascending=False)\n)\n\npd.set_option('display.max_rows', None)     # Show all rows\npd.set_option('display.max_columns', None)  # Show all columns\npd.set_option('display.width', 0)           # Auto-detect width\npd.set_option('display.max_colwidth', None) # Show full content in cells\n\nprint(xgb_fi_sorted.head(100))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:24:24.342130Z","iopub.status.idle":"2025-08-16T17:24:24.342646Z","shell.execute_reply.started":"2025-08-16T17:24:24.342339Z","shell.execute_reply":"2025-08-16T17:24:24.342353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import ndcg_score\nimport numpy as np\n\n# --- Separate LGB and XGB models from all_models ---\n#lgb_models = [m for (typ, _, m) in all_models if typ == 'lgb']\nxgb_models = [model for (typ, _, _, model) in all_models if typ == 'xgb']\n\n# --- Predict on validation set and average ---\nprint(\"\\n📊 Averaging predictions on validation set...\")\n\n# LightGBM predictions\n#preds_val_lgb = np.mean([model.predict(data_xgb_va) for model in lgb_models], axis=0)\n\n# XGBoost predictions\ndata_xgb_va_pd = data_xgb_va.drop('ranker_id').to_pandas()\ndval = xgb.DMatrix(data_xgb_va_pd)\npreds_val_xgb = np.mean([model.predict(dval) for model in xgb_models], axis=0)\n\n# Combine XGBoost predictions\npreds_val = preds_val_xgb\n\n# Convert labels to numeric if needed\ny_true = y_va['selected'].to_numpy().flatten()\n\n# Evaluate NDCG@3\nndcg_val = ndcg_score([y_true], [preds_val], k=3)\nprint(f\"✅ Ensemble NDCG@3: {ndcg_val:.4f}\")\n\n# Evaluate HitRate@3 (if function is defined)\nensemble_hr3 = hitrate_at_3(y_true, preds_val, groups_va['ranker_id'])\nprint(f\"🎯 Ensemble HitRate@3: {ensemble_hr3:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:24:24.343900Z","iopub.status.idle":"2025-08-16T17:24:24.344581Z","shell.execute_reply.started":"2025-08-16T17:24:24.344056Z","shell.execute_reply":"2025-08-16T17:24:24.344069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, penalty_factor=0.12):\n    COLS_TO_COMPARE = [\n        \"legs0_departureAt\",\n        \"legs0_arrivalAt\",\n        \"legs1_departureAt\",\n        \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\",\n        \"legs1_segments0_flightNumber\",\n        \"legs0_segments0_aircraft_code\",\n        \"legs1_segments0_aircraft_code\",\n        \"legs0_segments0_departureFrom_airport_iata\",\n        \"legs1_segments0_departureFrom_airport_iata\",\n    ]\n\n    test = test.with_columns(\n        [pl.col(c).cast(str).fill_null(\"NULL\") for c in COLS_TO_COMPARE]\n    )\n\n    df = submission_xgb.join(test, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    df = df.with_columns(\n        (\n            pl.col(\"legs0_departureAt\")\n            + \"_\"\n            + pl.col(\"legs0_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs1_departureAt\")\n            + \"_\"\n            + pl.col(\"legs1_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs0_segments0_flightNumber\")\n            + \"_\"\n            + pl.col(\"legs1_segments0_flightNumber\")\n        ).alias(\"flight_hash\")\n    )\n\n    df = df.with_columns(\n        pl.max(\"pred_score\")\n        .over([\"ranker_id\", \"flight_hash\"])\n        .alias(\"max_score_same_flight\")\n    )\n\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\")\n            - penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\"))\n        ).alias(\"reorder_score\")\n    )\n\n    df = df.with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:24:24.345400Z","iopub.status.idle":"2025-08-16T17:24:24.345940Z","shell.execute_reply.started":"2025-08-16T17:24:24.345532Z","shell.execute_reply":"2025-08-16T17:24:24.345544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Predict on test set ---\nprint(\"\\n📦 Generating predictions on test set...\")\n\n# # XGBoost test predictions\n\ndata_xgb_te_pd = data_xgb_te.drop('ranker_id').to_pandas()\ndtest = xgb.DMatrix(data_xgb_te_pd)\npreds_test_xgb = np.mean([model.predict(dtest) for model in xgb_models], axis=0)\n\n# Combined test ensemble\nensemble_test_preds =  preds_test_xgb\n\nsubmission_df = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', ensemble_test_preds))\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n)\n\ntop = re_rank(test, submission_df)\n\nsubmission_df = (\n    submission_df.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n    .with_columns(\n        [\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ]\n    )\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\n\n# --- Save to CSV ---\nsubmission_df.write_csv('submission.csv')\nprint(\"\\n✅ Submission file 'submission.csv' created successfully.\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T17:24:24.346753Z","iopub.status.idle":"2025-08-16T17:24:24.347221Z","shell.execute_reply.started":"2025-08-16T17:24:24.346903Z","shell.execute_reply":"2025-08-16T17:24:24.346915Z"}},"outputs":[],"execution_count":null}]}