{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install -U xgboost\n!pip install -U polars","metadata":{"_uuid":"34c3ffce-c494-475d-98ec-186a1498a1db","_cell_guid":"7a9cb62b-ea30-47ba-8f9c-1dccad7cfb25","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:26:28.737856Z","iopub.execute_input":"2025-07-24T20:26:28.738125Z","iopub.status.idle":"2025-07-24T20:26:53.936812Z","shell.execute_reply.started":"2025-07-24T20:26:28.738100Z","shell.execute_reply":"2025-07-24T20:26:53.930627Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"_uuid":"05ec6e10-5cb2-404e-9eff-72d4369ea909","_cell_guid":"660565f7-c052-46c0-a832-767e76ef2b03","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:26:53.939108Z","iopub.execute_input":"2025-07-24T20:26:53.939341Z","iopub.status.idle":"2025-07-24T20:26:59.144857Z","shell.execute_reply.started":"2025-07-24T20:26:53.939316Z","shell.execute_reply":"2025-07-24T20:26:59.139663Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))","metadata":{"_uuid":"97337a10-befa-486b-8a25-49ba4e0da881","_cell_guid":"071948b3-d81b-4309-94ee-24390267e7f3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:26:59.146917Z","iopub.execute_input":"2025-07-24T20:26:59.147275Z","iopub.status.idle":"2025-07-24T20:27:03.532446Z","shell.execute_reply.started":"2025-07-24T20:26:59.147250Z","shell.execute_reply":"2025-07-24T20:27:03.527457Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helpers","metadata":{"_uuid":"67080232-c848-42e4-b730-2075f87da2bc","_cell_guid":"6834d124-5f9d-4406-aa21-e969ef48dce1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"_uuid":"c01fcce9-3fb9-4ce6-b74f-8df0b7955efd","_cell_guid":"7edb88ae-e07b-40d6-9c3d-6e0b5c7163f1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:27:03.534165Z","iopub.execute_input":"2025-07-24T20:27:03.534384Z","iopub.status.idle":"2025-07-24T20:27:03.543346Z","shell.execute_reply.started":"2025-07-24T20:27:03.534362Z","shell.execute_reply":"2025-07-24T20:27:03.540123Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{"_uuid":"9c7d4c83-6633-4796-a035-f4441b8e46b5","_cell_guid":"33518709-bcce-4af8-b3f0-77782b4cf5c6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\n\n# Combine all initial transformations\ndf = df.with_columns([\n        # Price features\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # Duration features\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n        \n        # Trip type\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # Total segments count\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # FF features\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # Binary features\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n        # Baggage & fees\n        (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n         pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n        (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + \n         pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n        \n        # Routes & carriers\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\", \"MOWAER/AERMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # Cabin\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n])\n\n# Segment counts - more efficient\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n    else:\n        seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n# Add segment-based features\n# First create segment counts\ndf = df.with_columns(seg_exprs)\n\n# Then use them for derived features\ndf = df.with_columns([\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n])\n\n# More derived features\ndf = df.with_columns([\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    (pl.col(\"baggage_total\") > 0).cast(pl.Int32).alias(\"has_baggage\"),\n    (pl.col(\"total_fees\") > 0).cast(pl.Int32).alias(\"has_fees\"),\n    (pl.col(\"total_fees\") / (pl.col(\"totalPrice\") + 1)).alias(\"fee_rate\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n])\n\n# Add major carrier flag if column exists\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns(\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\", \"U6\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\")\n    )\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\ndf = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n# Time features - batch process\ntime_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# Batch rank computations - more efficient with single pass\n# First apply the columns that will be used for ranking\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n])\n\n# Price and duration basic ranks\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n# Price-specific features\nprice_exprs = [\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n]\n\n# Apply initial ranks\ndf = df.with_columns(rank_exprs + price_exprs)\n\n# Cheapest direct - more efficient\ndirect_cheapest = (\n    df.filter(pl.col(\"is_direct_leg0\") == 1)\n    .group_by(\"ranker_id\")\n    .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n)\n\ndf = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n    ((pl.col(\"is_direct_leg0\") == 1) & \n     (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n).drop(\"min_direct\")\n\n# Popularity features - efficient join\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n    ])\n)\n\n# Final features including popularity\ndf = df.with_columns([\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n])","metadata":{"_uuid":"b6612526-2ccd-4113-b5bf-51e344f5dc52","_cell_guid":"eaedf6e8-03bb-4dde-8ad9-2780d816d243","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:27:03.546649Z","iopub.execute_input":"2025-07-24T20:27:03.546872Z","iopub.status.idle":"2025-07-24T20:28:10.705513Z","shell.execute_reply.started":"2025-07-24T20:27:03.546851Z","shell.execute_reply":"2025-07-24T20:28:10.700506Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Popularity feature based on round trip combination\nif all(col in df.columns for col in [\n    \"legs0_segments0_departureFrom_airport_iata\",\n    \"legs0_segments0_arrivalTo_airport_iata\",\n    \"legs1_segments0_departureFrom_airport_iata\",\n    \"legs1_segments0_arrivalTo_airport_iata\"\n]):\n    df = df.with_columns([\n        (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n         pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\")\n    ])\n\n    # Calculate frequency\n    round_trip_freq = (\n        train.with_columns([\n            (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n             pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\")\n        ])\n        .group_by(\"round_trip_route\")\n        .agg(pl.count().alias(\"rt_route_count\"))\n    )\n\n    df = df.join(round_trip_freq, on=\"round_trip_route\", how=\"left\").with_columns(\n        pl.col(\"rt_route_count\").fill_null(0).alias(\"round_trip_freq\")\n    ).drop(\"round_trip_route\")\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"round_trip_freq\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:28:10.707032Z","iopub.execute_input":"2025-07-24T20:28:10.707274Z","iopub.status.idle":"2025-07-24T20:28:14.781658Z","shell.execute_reply.started":"2025-07-24T20:28:10.707251Z","shell.execute_reply":"2025-07-24T20:28:14.777192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill nulls\ndata = df.with_columns(\n    [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n    [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n)","metadata":{"_uuid":"2c5b472c-2d7a-4808-b739-68b674b6973e","_cell_guid":"b1ab23b5-f22a-4e07-b302-2d1f4f4ba83e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:28:14.784112Z","iopub.execute_input":"2025-07-24T20:28:14.784403Z","iopub.status.idle":"2025-07-24T20:28:16.470669Z","shell.execute_reply.started":"2025-07-24T20:28:14.784376Z","shell.execute_reply":"2025-07-24T20:28:16.465593Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Selection","metadata":{"_uuid":"2ae6cb3d-0d0e-4acc-b9d4-a111f71ed4ee","_cell_guid":"85c77b79-d74e-46b6-8694-f339a7c2a7d8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Categorical features\ncat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber',\n]\n\n# Columns to exclude (uninformative or problematic)\nexclude_cols = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    #'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n    'frequentFlyer',  # Already processed\n    # Exclude constant columns\n    'pricingInfo_passengerCount'\n]\n\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols)\ny = data.select('selected')\ngroups = data.select('ranker_id')","metadata":{"_uuid":"c2411b9d-e956-41c2-acd9-387932aaa59e","_cell_guid":"1d5417c6-fbbe-48de-8b5a-f11a4cd99286","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:28:16.473369Z","iopub.execute_input":"2025-07-24T20:28:16.473651Z","iopub.status.idle":"2025-07-24T20:28:16.490896Z","shell.execute_reply.started":"2025-07-24T20:28:16.473612Z","shell.execute_reply":"2025-07-24T20:28:16.487126Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Training","metadata":{"_uuid":"f20e6e08-bf01-45f4-b653-1ea25cb8030c","_cell_guid":"11e37895-9392-44b4-8e12-09e60e00f587","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"data_xgb = X.with_columns([(pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int32) for c in cat_features_final])\n\nn1 = 16487352 # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n1], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n1], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n1], groups[n1:n2], groups[n2:]\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ndtrain = xgb.DMatrix(data_xgb_tr, label=y_tr, group=group_sizes_tr, feature_names=data_xgb.columns)\ndval   = xgb.DMatrix(data_xgb_va, label=y_va, group=group_sizes_va, feature_names=data_xgb.columns)\ndtest  = xgb.DMatrix(data_xgb_te, label=y_te, group=group_sizes_te, feature_names=data_xgb.columns)","metadata":{"_uuid":"67b41a2a-7e9d-4e89-9e21-24ebec8bd620","_cell_guid":"c68b7052-f0be-413e-b39c-40b641b26ba1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:28:16.494256Z","iopub.execute_input":"2025-07-24T20:28:16.494523Z","iopub.status.idle":"2025-07-24T20:29:06.463786Z","shell.execute_reply.started":"2025-07-24T20:28:16.494498Z","shell.execute_reply":"2025-07-24T20:29:06.458465Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"!pip install -U optuna\nimport optuna\nfrom sklearn.metrics import ndcg_score\n\ndef objective(trial):\n    params = {\n        'objective': 'rank:pairwise',\n        'eval_metric': 'ndcg@3',\n        'max_depth': trial.suggest_int('max_depth', 4, 12),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.2, log=True),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n        'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n        'lambda': trial.suggest_float('lambda', 1.0, 50.0),\n        'random_state': RANDOM_STATE,\n        'n_jobs': -1,\n    }\n\n    model = xgb.train(\n        params,\n        dtrain,\n        num_boost_round=800,\n        evals=[(dval, \"val\")],\n        early_stopping_rounds=50,\n        verbose_eval=False\n    )\n\n    val_preds = model.predict(dval)\n    score = hitrate_at_3(y_va, val_preds, groups_va)\n    return score  # maximize hitrate@3\n\nstudy = optuna.create_study(direction=\"maximize\", study_name=\"xgb_ranker_opt\")\nstudy.optimize(objective, n_trials=30)\n\nprint(\"Best score:\", study.best_value)\nprint(\"Best params:\", study.best_params)\n\n# Train XGBoost model\nxgb_model = xgb.train(\n    study.best_params,\n    dtrain,\n    num_boost_round=800,\n    evals=[(dtrain, 'train'), (dval, 'val')],\n    early_stopping_rounds=50, #\n    verbose_eval=50\n)\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T20:29:06.466175Z","iopub.execute_input":"2025-07-24T20:29:06.466441Z","iopub.status.idle":"2025-07-24T20:29:06.483493Z","shell.execute_reply.started":"2025-07-24T20:29:06.466415Z","shell.execute_reply":"2025-07-24T20:29:06.478998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBoost parameters\nxgb_params = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n    # 'device': 'cuda'\n}\n\n# Train XGBoost model\nprint(\"Training XGBoost model...\")\nxgb_model = xgb.train(\n    xgb_params,\n    dtrain,\n    num_boost_round=1000,\n    evals=[(dtrain, 'train'), (dval, 'val')],\n#     early_stopping_rounds=100,\n    verbose_eval=50\n)","metadata":{"_uuid":"9833f5a3-4382-4f1e-9035-c85071a52882","_cell_guid":"522a61ed-654b-4f58-ac14-a2e002d0f204","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:29:06.486003Z","iopub.execute_input":"2025-07-24T20:29:06.486220Z","iopub.status.idle":"2025-07-24T20:46:05.614491Z","shell.execute_reply.started":"2025-07-24T20:29:06.486199Z","shell.execute_reply":"2025-07-24T20:46:05.608599Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate XGBoost\nxgb_va_preds = xgb_model.predict(dval)\nxgb_hr3 = hitrate_at_3(y_va, xgb_va_preds, groups_va)\nprint(f\"HitRate@3: {xgb_hr3:.3f}\")","metadata":{"_uuid":"870bc922-cc2e-4514-881b-1eed55ba354a","_cell_guid":"0c2da8d1-533d-4946-98a0-63de106c9b10","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:46:05.616285Z","iopub.execute_input":"2025-07-24T20:46:05.616504Z","iopub.status.idle":"2025-07-24T20:46:07.681097Z","shell.execute_reply.started":"2025-07-24T20:46:05.616483Z","shell.execute_reply":"2025-07-24T20:46:07.675785Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_importance = xgb_model.get_score(importance_type='gain')\nxgb_importance_df = pl.DataFrame(\n    [{'feature': k, 'importance': v} for k, v in xgb_importance.items()]\n).sort('importance', descending=bool(1))\nprint(xgb_importance_df.head(20).to_pandas().to_string())","metadata":{"_uuid":"8e4be8da-0659-4dd0-b5bb-bf223c10ce1f","_cell_guid":"bb7fced5-388c-480c-824f-f40945f953da","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:46:07.682709Z","iopub.execute_input":"2025-07-24T20:46:07.682952Z","iopub.status.idle":"2025-07-24T20:46:07.726820Z","shell.execute_reply.started":"2025-07-24T20:46:07.682930Z","shell.execute_reply":"2025-07-24T20:46:07.722378Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Error analysis and visualization","metadata":{"_uuid":"661b71ce-1d41-4ae9-839a-6437d6521393","_cell_guid":"3b9d1a54-d272-4752-8e61-1acdfacde7e0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Color palette\nred = (0.86, 0.08, 0.24)\nblue = (0.12, 0.56, 1.0)\n\n# Prepare data for analysis\nva_df = pl.DataFrame({\n    'ranker_id': groups_va.to_numpy().flatten(),\n    'pred_score': xgb_va_preds,\n    'selected': y_va.to_numpy().flatten()\n})\n\n# Add group size and filter\nva_df = va_df.join(\n    va_df.group_by('ranker_id').agg(pl.len().alias('group_size')), \n    on='ranker_id'\n).filter(pl.col('group_size') > 10)\n\n# Calculate group size quantiles\nsize_quantiles = va_df.select('ranker_id', 'group_size').unique().select(\n    pl.col('group_size').quantile(0.25).alias('q25'),\n    pl.col('group_size').quantile(0.50).alias('q50'),\n    pl.col('group_size').quantile(0.75).alias('q75')\n).to_dicts()[0]\n\n# Function to calculate hitrate curve efficiently\ndef calculate_hitrate_curve(df, k_values):\n    # Sort once and calculate all k values\n    sorted_df = df.sort([\"ranker_id\", \"pred_score\"], descending=[False, True])\n    return [\n        sorted_df.group_by(\"ranker_id\", maintain_order=True)\n        .head(k)\n        .group_by(\"ranker_id\")\n        .agg(pl.col(\"selected\").max().alias(\"hit\"))\n        .select(pl.col(\"hit\").mean())\n        .item()\n        for k in k_values\n    ]\n\n# Calculate curves\nk_values = list(range(1, 21))\ncurves = {\n    'All groups (>10)': calculate_hitrate_curve(va_df, k_values),\n    f'Small (11-{int(size_quantiles[\"q25\"])})': calculate_hitrate_curve(\n        va_df.filter(pl.col('group_size') <= size_quantiles['q25']), k_values\n    ),\n    f'Medium ({int(size_quantiles[\"q25\"]+1)}-{int(size_quantiles[\"q75\"])})': calculate_hitrate_curve(\n        va_df.filter((pl.col('group_size') > size_quantiles['q25']) & \n                    (pl.col('group_size') <= size_quantiles['q75'])), k_values\n    ),\n    f'Large (>{int(size_quantiles[\"q75\"])})': calculate_hitrate_curve(\n        va_df.filter(pl.col('group_size') > size_quantiles['q75']), k_values\n    )\n}\n\n# Calculate hitrate@3 by group size using log-scale bins\n# Create log-scale bins\nmin_size = va_df['group_size'].min()\nmax_size = va_df['group_size'].max()\nbins = np.logspace(np.log10(min_size), np.log10(max_size), 51)  # 51 edges = 50 bins\n\n# Calculate hitrate@3 for each ranker_id\nranker_hr3 = (\n    va_df.sort([\"ranker_id\", \"pred_score\"], descending=[False, True])\n    .group_by(\"ranker_id\", maintain_order=True)\n    .agg([\n        pl.col(\"selected\").head(3).max().alias(\"hit_top3\"),\n        pl.col(\"group_size\").first()\n    ])\n)\n\n# Assign bins and calculate hitrate per bin\nbin_centers = (bins[:-1] + bins[1:]) / 2  # Geometric mean would be more accurate for log scale\nbin_indices = np.digitize(ranker_hr3['group_size'].to_numpy(), bins) - 1\n\nsize_analysis = pl.DataFrame({\n    'bin_idx': bin_indices,\n    'bin_center': bin_centers[np.clip(bin_indices, 0, len(bin_centers)-1)],\n    'hit_top3': ranker_hr3['hit_top3']\n}).group_by(['bin_idx', 'bin_center']).agg([\n    pl.col('hit_top3').mean().alias('hitrate3'),\n    pl.len().alias('n_groups')\n]).filter(pl.col('n_groups') >= 3).sort('bin_center')  # At least 3 groups per bin\n\n# Create combined figure\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4), dpi=400)\n\n# Left plot: HitRate@k curves\n# Create color gradient from blue to red for size groups\ncolors = ['black']  # All groups is black\nfor i in range(3):  # 3 size groups\n    t = i / 2  # 0, 0.5, 1\n    color = tuple(blue[j] * (1 - t) + red[j] * t for j in range(3))\n    colors.append(color)\n\nfor (label, hitrates), color in zip(curves.items(), colors):\n    ax1.plot(k_values, hitrates, marker='o', label=label, color=color, markersize=3)\nax1.set_xlabel('k (top-k predictions)')\nax1.set_ylabel('HitRate@k')\nax1.set_title('HitRate@k by Group Size')\nax1.legend(fontsize=8)\nax1.grid(True, alpha=0.3)\nax1.set_xlim(0, 21)\nax1.set_ylim(-0.025, 1.025)\n\n# Right plot: HitRate@3 vs Group Size (log scale)\nax2.scatter(size_analysis['bin_center'], size_analysis['hitrate3'], s=30, alpha=0.6, color=blue)\nax2.set_xlabel('Group Size')\nax2.set_ylabel('HitRate@3')\nax2.set_title('HitRate@3 vs Group Size')\nax2.set_xscale('log')\nax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"_uuid":"8f84a3e1-b456-4eec-8327-aaa217380e8d","_cell_guid":"95c07b5d-a2de-49c5-a0d4-ef28c001a3dd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:46:07.729345Z","iopub.execute_input":"2025-07-24T20:46:07.729553Z","iopub.status.idle":"2025-07-24T20:46:10.738874Z","shell.execute_reply.started":"2025-07-24T20:46:07.729532Z","shell.execute_reply":"2025-07-24T20:46:10.732186Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Summary\nprint(f\"HitRate@1: {curves['All groups (>10)'][0]:.3f}\")\nprint(f\"HitRate@3: {curves['All groups (>10)'][2]:.3f}\")\nprint(f\"HitRate@5: {curves['All groups (>10)'][4]:.3f}\")\nprint(f\"HitRate@10: {curves['All groups (>10)'][9]:.3f}\")","metadata":{"_uuid":"1e5ae1a3-2d94-481c-99fc-dfc0da1f37ce","_cell_guid":"27b691ad-a4b1-40bb-9117-82f902c74e35","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:46:10.739717Z","iopub.execute_input":"2025-07-24T20:46:10.739968Z","iopub.status.idle":"2025-07-24T20:46:10.750645Z","shell.execute_reply.started":"2025-07-24T20:46:10.739943Z","shell.execute_reply":"2025-07-24T20:46:10.745717Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{"_uuid":"1aaa668b-cd71-4f4e-a160-1ed6187f16d5","_cell_guid":"f27e0170-0bb1-41d3-b747-2e476f7dd428","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"submission_xgb = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', xgb_model.predict(dtest)))\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected'])\n)\nsubmission_xgb.write_csv('submission.csv')","metadata":{"_uuid":"2451e3b1-3062-4cab-88e9-e44d635b5be6","_cell_guid":"a5f8c612-bc01-43c4-86ba-ceaffae59a8d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:46:10.752276Z","iopub.execute_input":"2025-07-24T20:46:10.752486Z","iopub.status.idle":"2025-07-24T20:46:20.180523Z","shell.execute_reply.started":"2025-07-24T20:46:10.752465Z","shell.execute_reply":"2025-07-24T20:46:20.175896Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}