{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U xgboost\n!pip install -U polars\n!pip install -U optuna\n!pip install -U catboost\n!pip install -U lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:03.383975Z","iopub.execute_input":"2025-08-15T10:46:03.384311Z","iopub.status.idle":"2025-08-15T10:46:46.039259Z","shell.execute_reply.started":"2025-08-15T10:46:03.384283Z","shell.execute_reply":"2025-08-15T10:46:46.034704Z"}},"outputs":[{"name":"stdout","text":"Collecting xgboost\n  Downloading xgboost-3.0.4-py3-none-manylinux_2_28_x86_64.whl (94.9 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.9/94.9 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from xgboost) (2.0.2)\nRequirement already satisfied: nvidia-nccl-cu12 in /usr/local/lib/python3.10/site-packages (from xgboost) (2.21.5)\nRequirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from xgboost) (1.15.3)\nInstalling collected packages: xgboost\nSuccessfully installed xgboost-3.0.4\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\nCollecting polars\n  Downloading polars-1.32.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.4 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.4/38.4 MB\u001b[0m \u001b[31m25.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: polars\nSuccessfully installed polars-1.32.3\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\nCollecting optuna\n  Downloading optuna-4.4.0-py3-none-any.whl (395 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m395.9/395.9 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/site-packages (from optuna) (4.67.1)\nRequirement already satisfied: PyYAML in /usr/local/lib/python3.10/site-packages (from optuna) (6.0.2)\nCollecting sqlalchemy>=1.4.2\n  Downloading sqlalchemy-2.0.43-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.2 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting colorlog\n  Downloading colorlog-6.9.0-py3-none-any.whl (11 kB)\nRequirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from optuna) (2.0.2)\nCollecting alembic>=1.5.0\n  Downloading alembic-1.16.4-py3-none-any.whl (247 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.0/247.0 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from optuna) (25.0)\nRequirement already satisfied: tomli in /usr/local/lib/python3.10/site-packages (from alembic>=1.5.0->optuna) (2.2.1)\nRequirement already satisfied: typing-extensions>=4.12 in /usr/local/lib/python3.10/site-packages (from alembic>=1.5.0->optuna) (4.14.0)\nCollecting Mako\n  Downloading mako-1.3.10-py3-none-any.whl (78 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.5/78.5 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting greenlet>=1\n  Downloading greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl (584 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m584.4/584.4 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/site-packages (from Mako->alembic>=1.5.0->optuna) (3.0.2)\nInstalling collected packages: Mako, greenlet, colorlog, sqlalchemy, alembic, optuna\nSuccessfully installed Mako-1.3.10 alembic-1.16.4 colorlog-6.9.0 greenlet-3.2.4 optuna-4.4.0 sqlalchemy-2.0.43\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\nCollecting catboost\n  Downloading catboost-1.2.8-cp310-cp310-manylinux2014_x86_64.whl (99.2 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.2/99.2 MB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.10/site-packages (from catboost) (1.17.0)\nRequirement already satisfied: pandas>=0.24 in /usr/local/lib/python3.10/site-packages (from catboost) (2.3.0)\nRequirement already satisfied: matplotlib in /usr/local/lib/python3.10/site-packages (from catboost) (3.10.3)\nRequirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from catboost) (1.15.3)\nCollecting graphviz\n  Downloading graphviz-0.21-py3-none-any.whl (47 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.3/47.3 kB\u001b[0m \u001b[31m996.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[?25hCollecting plotly\n  Downloading plotly-6.3.0-py3-none-any.whl (9.8 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.8/9.8 MB\u001b[0m \u001b[31m57.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: numpy<3.0,>=1.16.0 in /usr/local/lib/python3.10/site-packages (from catboost) (2.0.2)\nRequirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/site-packages (from pandas>=0.24->catboost) (2025.2)\nRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas>=0.24->catboost) (2025.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas>=0.24->catboost) (2.9.0.post0)\nRequirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (4.58.4)\nRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (25.0)\nRequirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (0.12.1)\nRequirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (1.4.8)\nRequirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (3.2.3)\nRequirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (1.3.2)\nRequirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/site-packages (from matplotlib->catboost) (11.3.0)\nCollecting narwhals>=1.15.1\n  Downloading narwhals-2.1.2-py3-none-any.whl (392 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m392.1/392.1 kB\u001b[0m \u001b[31m27.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hInstalling collected packages: narwhals, graphviz, plotly, catboost\nSuccessfully installed catboost-1.2.8 graphviz-0.21 narwhals-2.1.2 plotly-6.3.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\nCollecting lightgbm\n  Downloading lightgbm-4.6.0-py3-none-manylinux_2_28_x86_64.whl (3.6 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from lightgbm) (1.15.3)\nRequirement already satisfied: numpy>=1.17.0 in /usr/local/lib/python3.10/site-packages (from lightgbm) (2.0.2)\nInstalling collected packages: lightgbm\nSuccessfully installed lightgbm-4.6.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:46.041275Z","iopub.execute_input":"2025-08-15T10:46:46.04156Z","iopub.status.idle":"2025-08-15T10:46:46.498125Z","shell.execute_reply.started":"2025-08-15T10:46:46.04152Z","shell.execute_reply":"2025-08-15T10:46:46.493914Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/aeroclub-recsys-2025/jsons_raw.tar.kaggle\n/kaggle/input/aeroclub-recsys-2025/train.parquet\n/kaggle/input/aeroclub-recsys-2025/sample_submission.parquet\n/kaggle/input/aeroclub-recsys-2025/jsons_structure.md\n/kaggle/input/aeroclub-recsys-2025/test.parquet\n","output_type":"stream"}],"execution_count":2},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nimport catboost\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import ndcg_score\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:46.499921Z","iopub.execute_input":"2025-08-15T10:46:46.500216Z","iopub.status.idle":"2025-08-15T10:46:51.933802Z","shell.execute_reply.started":"2025-08-15T10:46:46.500191Z","shell.execute_reply":"2025-08-15T10:46:51.92933Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n","output_type":"stream"}],"execution_count":3},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:51.936114Z","iopub.execute_input":"2025-08-15T10:46:51.936434Z","iopub.status.idle":"2025-08-15T10:46:56.170376Z","shell.execute_reply.started":"2025-08-15T10:46:51.936411Z","shell.execute_reply":"2025-08-15T10:46:56.164473Z"}},"outputs":[],"execution_count":4},{"cell_type":"code","source":"top_hubs = (\n    train.select(\"legs0_segments1_departureFrom_airport_iata\")\n         .drop_nulls()\n         .group_by(\"legs0_segments1_departureFrom_airport_iata\")\n         .agg(pl.count().alias(\"count\"))\n         .sort(\"count\", descending=True)\n         .head(10)\n)\n\nhub_airports = top_hubs[\"legs0_segments1_departureFrom_airport_iata\"].to_list()\nhub_airports","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:56.172231Z","iopub.execute_input":"2025-08-15T10:46:56.172479Z","iopub.status.idle":"2025-08-15T10:46:56.329024Z","shell.execute_reply.started":"2025-08-15T10:46:56.172458Z","shell.execute_reply":"2025-08-15T10:46:56.323623Z"}},"outputs":[{"name":"stderr","text":"/tmp/ipykernel_10/1324459448.py:5: DeprecationWarning: `pl.count()` is deprecated. Please use `pl.len()` instead.\n(Deprecated in version 0.20.5)\n  .agg(pl.count().alias(\"count\"))\n","output_type":"stream"},{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"['SVO', 'IST', 'OVB', 'SVX', 'DME', 'DXB', 'LED', 'AER', 'PEK', 'IKT']"},"metadata":{}}],"execution_count":5},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:56.330542Z","iopub.execute_input":"2025-08-15T10:46:56.330961Z","iopub.status.idle":"2025-08-15T10:46:56.341727Z","shell.execute_reply.started":"2025-08-15T10:46:56.330935Z","shell.execute_reply":"2025-08-15T10:46:56.33626Z"}},"outputs":[],"execution_count":6},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\n\n# Combine all initial transformations\ndf = df.with_columns([\n        # Price features\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # Duration features\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n        \n        # Trip type\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # Total segments count\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # FF features\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # Binary features\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n        # Baggage & fees\n        # (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n        #  pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n        # (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + \n        #  pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n\n        (\n            (pl.col(\"miniRules0_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules0_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_cancel\"),\n        (\n            (pl.col(\"miniRules1_monetaryAmount\") == 0)\n            & (pl.col(\"miniRules1_statusInfos\") == 1)\n        )\n        .cast(pl.Int8)\n        .alias(\"free_exchange\"),\n    \n        # Routes & carriers\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # Cabin\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n])\n\n# Segment counts - more efficient\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n    else:\n        seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n\n# Add segment-based features\n# First create segment counts\ndf = df.with_columns(seg_exprs)\n\n# Then use them for derived features\ndf = df.with_columns([\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n])\n\n# More derived features\ndf = df.with_columns([\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    # (pl.col(\"baggage_total\") > 0).cast(pl.Int32).alias(\"has_baggage\"),\n    # (pl.col(\"total_fees\") > 0).cast(pl.Int32).alias(\"has_fees\"),\n    # (pl.col(\"total_fees\") / (pl.col(\"totalPrice\") + 1)).alias(\"fee_rate\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n])\n\n# Add major carrier flag if column exists\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns(\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\")\n    )\nelse:\n    df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n\ndf = df.with_columns(pl.col(\"group_size\").log1p().alias(\"group_size_log\"))\n\n# Time features - batch process\ntime_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# Batch rank computations - more efficient with single pass\n# First apply the columns that will be used for ranking\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n])\n\n# Price and duration basic ranks\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n\n# Price-specific features\nprice_exprs = [\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n]\n\n# Apply initial ranks\ndf = df.with_columns(rank_exprs + price_exprs)\n\n# Cheapest direct - more efficient\ndirect_cheapest = (\n    df.filter(pl.col(\"is_direct_leg0\") == 1)\n    .group_by(\"ranker_id\")\n    .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n)\n\ndf = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\").with_columns(\n    ((pl.col(\"is_direct_leg0\") == 1) & \n     (pl.col(\"totalPrice\") == pl.col(\"min_direct\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n).drop(\"min_direct\")\n\n# Popularity features - efficient join\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier0_pop')),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg(pl.mean('selected').alias('carrier1_pop')),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n    ])\n)\n\n# Step 1: Add independent features\ndf = df.with_columns([\n    # Carrier popularity\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n])\n\ndf = df.with_columns(\n    (\n        # Policy compliance (25% weight)\n        (pl.col(\"pricingInfo_isAccessTP\") * 0.25) +\n        # Direct flights (25% weight)\n        (pl.col(\"is_direct_leg0\") * 0.25) +\n        # Business-hour departures/arrivals (25% weight)\n        ((pl.col(\"legs0_departureAt_business_time\") + pl.col(\"legs1_departureAt_business_time\")) * 0.125) +\n        # VIP preference for business class (25% weight)\n        ((pl.col(\"isVip\") == 1) & (pl.col(\"avg_cabin_class\") >= 1.5)).cast(pl.Int8) * 0.25\n    ).alias(\"business_traveler_perfect_match\"),\n\n    # Timezone diff only\n    (pl.col(\"legs0_arrivalAt_hour\") - pl.col(\"legs0_departureAt_hour\") -\n     (pl.col(\"legs0_duration\") / 60)).alias(\"timezone_diff_leg0\"),\n)\n\ndf = df.with_columns(\n    (\n        (pl.col(\"is_one_way\") == 0) &  # Round-trip\n        (pl.col(\"legs0_arrivalAt_hour\") >= 8) &  # Arrive by morning\n        (pl.col(\"legs1_departureAt_hour\") <= 18) &  # Return by evening\n        (pl.col(\"timezone_diff_leg0\").abs() < 3)   # Minimal jetlag\n    ).cast(pl.Int8).alias(\"meeting_friendly_itinerary\")\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:46:56.343274Z","iopub.execute_input":"2025-08-15T10:46:56.343483Z","iopub.status.idle":"2025-08-15T10:48:04.111306Z","shell.execute_reply.started":"2025-08-15T10:46:56.343462Z","shell.execute_reply":"2025-08-15T10:48:04.106075Z"}},"outputs":[],"execution_count":7},{"cell_type":"code","source":"data = df.with_columns(\n    [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n    [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:04.113787Z","iopub.execute_input":"2025-08-15T10:48:04.11404Z","iopub.status.idle":"2025-08-15T10:48:06.465235Z","shell.execute_reply.started":"2025-08-15T10:48:04.114014Z","shell.execute_reply":"2025-08-15T10:48:06.459374Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"#df = df.with_columns([\n    #(df[\"legs0_segments0_departureFrom_airport_iata\"] == df[\"legs1_segments0_arrivalTo_airport_iata\"]).cast(pl.Int8).alias(\"roundtrip_symmetric\"),\n    #(df[\"carrier0_pop\"] - df[\"carrier1_pop\"]).fill_null(0).alias(\"carrier_popularity_diff\"),\n    #(1 / (df[\"group_size\"] + 1)).alias(\"inv_group_size\"),\n    # (pl.col(\"is_min_segments\") == True) & (pl.col(\"group_size\") <= 15).cast(pl.Int32).fill_null(0).alias(\"is_min_segmentand_group_size\")\n#])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:06.467874Z","iopub.execute_input":"2025-08-15T10:48:06.468148Z","iopub.status.idle":"2025-08-15T10:48:06.47684Z","shell.execute_reply.started":"2025-08-15T10:48:06.46812Z","shell.execute_reply":"2025-08-15T10:48:06.472457Z"}},"outputs":[],"execution_count":9},{"cell_type":"code","source":"cat_features = [\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber'\n]\n\n# Columns to exclude (uninformative or problematic)\nexclude_cols = [\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n    'frequentFlyer',  # Already processed\n    # Exclude constant columns\n    'pricingInfo_passengerCount','bySelf','n_segments_leg1','timezone_diff_leg0','meeting_friendly_itinerary','business_traveler_perfect_match'\n]\n\nfor leg in [0, 1]:\n    for seg in [0, 1]:\n        if seg == 0:\n            suffixes = [\n                \"seatsAvailable\",\n            ]\n        else:\n            suffixes = [\n                \"cabinClass\",\n                \"seatsAvailable\",\n                \"baggageAllowance_quantity\",\n                \"baggageAllowance_weightMeasurementType\",\n                \"aircraft_code\",\n                \"arrivalTo_airport_city_iata\",\n                \"arrivalTo_airport_iata\",\n                \"departureFrom_airport_iata\",\n                \"flightNumber\",\n                \"marketingCarrier_code\",\n                \"operatingCarrier_code\",\n            ]\n        for suffix in suffixes:\n            exclude_cols.append(f\"legs{leg}_segments{seg}_{suffix}\")\n\n\n# Exclude segment 2-3 columns (>98% missing)\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols + ['ranker_id'])\ny = data.select(['selected', 'ranker_id'])\ngroups = data.select('ranker_id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:06.479452Z","iopub.execute_input":"2025-08-15T10:48:06.479801Z","iopub.status.idle":"2025-08-15T10:48:06.502565Z","shell.execute_reply.started":"2025-08-15T10:48:06.479773Z","shell.execute_reply":"2025-08-15T10:48:06.497559Z"}},"outputs":[{"name":"stdout","text":"Using 86 features (19 categorical)\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"data_xgb = X.with_columns([(pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int16) for c in cat_features_final])\n\nn1 = 16487352 # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n2], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n2], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n2], groups[n1:n2], groups[n2:]\n\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\ngroup_sizes_tr_lgb = groups_tr.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_va_lgb = groups_va.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\ngroup_sizes_te_lgb = groups_te.group_by('ranker_id').agg(pl.len()).sort('ranker_id')['len'].to_numpy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:06.503925Z","iopub.execute_input":"2025-08-15T10:48:06.504161Z","iopub.status.idle":"2025-08-15T10:48:12.286818Z","shell.execute_reply.started":"2025-08-15T10:48:06.504137Z","shell.execute_reply":"2025-08-15T10:48:12.280332Z"}},"outputs":[],"execution_count":11},{"cell_type":"code","source":"# --- Step 1: group size & median ---\nranker_group_sizes = groups_tr.group_by('ranker_id').agg(pl.len().alias('group_size'))\nmedian_group_size = ranker_group_sizes['group_size'].median()\nprint(f\"Median group size: {median_group_size}\")\n\n# --- Step 2: get ranker_id lists ---\nsmall_rankers = ranker_group_sizes.filter(pl.col('group_size') < median_group_size)['ranker_id']\nbig_rankers   = ranker_group_sizes.filter(pl.col('group_size') >= median_group_size)['ranker_id']\n\n# --- Step 3: rankers where is_min_segments == 0 ---\nmin_segment_0 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 0)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nmin_segment_1 = (\n    data_xgb_tr\n    .filter(pl.col('is_min_segments') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nis_one_way_ranker = (\n    data_xgb_tr\n    .filter(pl.col('is_one_way') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\nis_popular_route_ranker = (\n    data_xgb_tr\n    .filter(pl.col('is_popular_route') == 1)\n    .select('ranker_id')\n    .unique()\n)['ranker_id']\n\n# --- Step 4: filtering helper ---\ndef split_data(ranker_ids):\n    ranker_list = ranker_ids.to_list() if hasattr(ranker_ids, \"to_list\") else list(ranker_ids)\n    data = data_xgb_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    y    = y_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    grp  = groups_tr.filter(pl.col('ranker_id').is_in(ranker_list))\n    return (\n        data.drop('ranker_id').to_pandas(),\n        y.drop('ranker_id')['selected'].to_numpy(),\n        grp\n    )\n\n\n# last_minute_rankers = (\n#     data_xgb_tr\n#     .with_columns(\n#         (pl.col(\"legs0_departureAt\").str.to_datetime() - pl.col(\"search_date\").str.to_datetime())\n#         .dt.days().alias(\"days_until_departure\")\n#     )\n#     .filter(pl.col(\"days_until_departure\") <= 3)\n#     .select(\"ranker_id\")\n#     .unique()[\"ranker_id\"]\n# )\n\n# 2. Cheapest flights in each search\ncheapest_rankers = (\n    data_xgb_tr\n    .filter(pl.col(\"is_cheapest\") == 1)\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 3. Direct flights only\ndirect_flight_rankers = (\n    data_xgb_tr\n    .filter((pl.col(\"is_direct_leg0\") == 1) & (pl.col(\"is_one_way\") == 1))\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 4. High loyalty passengers\nloyalty_rankers = (\n    data_xgb_tr\n    .filter(pl.col(\"n_ff_programs\") >= 2)\n    .select(\"ranker_id\")\n    .unique()[\"ranker_id\"]\n)\n\n# 5. Business-friendly itineraries\n# biz_friendly_rankers = (\n#     data_xgb_tr\n#     .filter(pl.col(\"business_traveler_perfect_match\") >= 0.75)\n#     .select(\"ranker_id\")\n#     .unique()[\"ranker_id\"]\n# )\n\n# --- Step Z: create new scenario datasets ---\ndata_xgb_tr_small_pd, y_tr_small_pd, groups_tr_small   = split_data(small_rankers)\ndata_xgb_tr_big_pd,   y_tr_big_pd,   groups_tr_big     = split_data(big_rankers)\ndata_xgb_tr_min0_pd,  y_tr_min0_pd,  groups_tr_min0    = split_data(min_segment_0)\ndata_xgb_tr_one_way_pd,  y_tr_one_way_pd,  groups_tr_one_way    = split_data(is_one_way_ranker)\ndata_xgb_tr_popular_route_pd,  y_tr_popular_route_pd,  groups_tr_popular_route_way    = split_data(is_popular_route_ranker)\n#data_xgb_tr_last_minute_pd, y_tr_last_minute_pd, groups_tr_last_minute = split_data(last_minute_rankers)\n#data_xgb_tr_cheapest_pd,   y_tr_cheapest_pd,   groups_tr_cheapest     = split_data(cheapest_rankers)\n#data_xgb_tr_direct_pd,     y_tr_direct_pd,     groups_tr_direct       = split_data(direct_flight_rankers)\ndata_xgb_tr_loyalty_pd,    y_tr_loyalty_pd,    groups_tr_loyalty      = split_data(loyalty_rankers)\n#data_xgb_tr_biz_pd,        y_tr_biz_pd,        groups_tr_biz          = split_data(biz_friendly_rankers)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:12.288974Z","iopub.execute_input":"2025-08-15T10:48:12.289265Z","iopub.status.idle":"2025-08-15T10:48:20.335114Z","shell.execute_reply.started":"2025-08-15T10:48:12.28924Z","shell.execute_reply":"2025-08-15T10:48:20.328286Z"}},"outputs":[{"name":"stdout","text":"Median group size: 50.0\n","output_type":"stream"}],"execution_count":12},{"cell_type":"code","source":"data_xgb_tr.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:20.336917Z","iopub.execute_input":"2025-08-15T10:48:20.337138Z","iopub.status.idle":"2025-08-15T10:48:20.3501Z","shell.execute_reply.started":"2025-08-15T10:48:20.337114Z","shell.execute_reply":"2025-08-15T10:48:20.3447Z"}},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"(18145372, 87)"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"print(\"helloooo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:20.353204Z","iopub.execute_input":"2025-08-15T10:48:20.353427Z","iopub.status.idle":"2025-08-15T10:48:20.362174Z","shell.execute_reply.started":"2025-08-15T10:48:20.353404Z","shell.execute_reply":"2025-08-15T10:48:20.357561Z"}},"outputs":[{"name":"stdout","text":"helloooo\n","output_type":"stream"}],"execution_count":14},{"cell_type":"code","source":"len(min_segment_1) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:20.364528Z","iopub.execute_input":"2025-08-15T10:48:20.364808Z","iopub.status.idle":"2025-08-15T10:48:20.379753Z","shell.execute_reply.started":"2025-08-15T10:48:20.364775Z","shell.execute_reply":"2025-08-15T10:48:20.37396Z"}},"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"105539"},"metadata":{}}],"execution_count":15},{"cell_type":"code","source":"import xgboost as xgb\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import ndcg_score\n\n# ----------------- Constants and Globals -----------------\nRANDOM_STATE = 42\nshap_lgbm_seed = {}\nxgb_seeds = [50]\nlgb_seeds = [42]\nall_models = []\nfeature_importances = []\n\n# ----------------- XGBoost Params -----------------\nxgb_rank_params = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': RANDOM_STATE,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_small = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 22,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_big = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_rank_params_min0 = {\n    'objective': 'rank:pairwise',\n    'eval_metric': 'ndcg@3',\n    \"learning_rate\": 0.022641389657079056,\n    \"max_depth\": 14,\n    \"min_child_weight\": 2,\n    \"subsample\": 0.8842234913702768,\n    \"colsample_bytree\": 0.45840689146263086,\n    \"gamma\": 3.3084297630544888,\n    \"lambda\": 6.952586917313028,\n    \"alpha\": 0.6395254133055179,\n    'seed': 19,\n    'n_jobs': -1,\n}\n\nxgb_params = {\n    'full':    xgb_rank_params,\n    'small':   xgb_rank_params,\n    'big':     xgb_rank_params,\n    'min0':    xgb_rank_params,\n    #'min1':    xgb_rank_params,\n    'one_way': xgb_rank_params,\n    #'last_minute': xgb_rank_params,\n    #'cheapest':    xgb_rank_params,\n    'loyalty':     xgb_rank_params,\n    'popular_route': xgb_rank_params,\n    #'biz':         xgb_rank_params\n}\n\n# ----------------- LightGBM Params -----------------\nlgb_rank_params = {\n    'objective': 'lambdarank',\n    'metric': 'ndcg',\n    'boosting_type': 'gbdt',\n    'eval_at': [3],\n    'num_leaves': 137,\n    'learning_rate': 0.1923609,\n    'min_child_samples': 69,\n    'lambda_l1': 0.0017863,\n    'lambda_l2': 7.8818,\n    'feature_fraction': 0.6015,\n    'bagging_fraction': 0.8536,\n    'bagging_freq': 7,\n    'verbosity': -1,\n    'label_gain': [0, 1]\n}\n\n# ----------------- Drop ranker_id from pandas frames -----------------\nfor df in [\n    data_xgb_tr_small_pd, data_xgb_tr_big_pd, data_xgb_tr_min0_pd,\n    data_xgb_tr_one_way_pd, data_xgb_tr_popular_route_pd,\n    data_xgb_tr_loyalty_pd\n]:\n    if 'ranker_id' in df.columns:\n        df.drop(columns=['ranker_id'], inplace=True)\n\n# ----------------- Group sizes for small/big -----------------\ngroup_sizes_tr_small        = groups_tr_small.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_big          = groups_tr_big.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_min_0         = groups_tr_min0.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_size_tr_min_1        = groups_tr_min1.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_one_way        = groups_tr_one_way.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_size_tr_popular_route        = groups_tr_popular_route_way.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_last_minute  = groups_tr_last_minute.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_cheapest     = groups_tr_cheapest.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_direct       = groups_tr_direct.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_tr_loyalty      = groups_tr_loyalty.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n#group_sizes_tr_biz          = groups_tr_biz.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\n# ----------------- DMatrix Setup -----------------\ndtrain_full = xgb.DMatrix(\n    data_xgb_tr.drop('ranker_id').to_pandas(),\n    label=y_tr['selected'].to_numpy(),\n    group=group_sizes_tr,\n    feature_names=data_xgb.drop('ranker_id').columns\n)\n\ndtrain_small = xgb.DMatrix(data_xgb_tr_small_pd, label=y_tr_small_pd, group=group_sizes_tr_small, feature_names=list(data_xgb_tr_small_pd.columns))\ndtrain_big   = xgb.DMatrix(data_xgb_tr_big_pd,   label=y_tr_big_pd,   group=group_sizes_tr_big,   feature_names=list(data_xgb_tr_big_pd.columns))\ndtrain_min_0 = xgb.DMatrix(data_xgb_tr_min0_pd,  label=y_tr_min0_pd,  group=group_size_tr_min_0,  feature_names=list(data_xgb_tr_min0_pd.columns))\n#dtrain_min_1 = xgb.DMatrix(data_xgb_tr_min1_pd,  label=y_tr_min1_pd,  group=group_size_tr_min_1,  feature_names=list(data_xgb_tr_min1_pd.columns))\ndtrain_one_way = xgb.DMatrix(data_xgb_tr_one_way_pd,  label=y_tr_one_way_pd,  group=group_size_tr_one_way,  feature_names=list(data_xgb_tr_one_way_pd.columns))\ndtrain_popular_route = xgb.DMatrix(data_xgb_tr_popular_route_pd,  label=y_tr_popular_route_pd,  group=group_size_tr_popular_route,  feature_names=list(data_xgb_tr_popular_route_pd.columns))\n\n# New scenarios\n#dtrain_last_minute = xgb.DMatrix(data_xgb_tr_last_minute_pd, label=y_tr_last_minute_pd, group=group_sizes_tr_last_minute, feature_names=list(data_xgb_tr_last_minute_pd.columns))\n#dtrain_cheapest    = xgb.DMatrix(data_xgb_tr_cheapest_pd,    label=y_tr_cheapest_pd,    group=group_sizes_tr_cheapest,    feature_names=list(data_xgb_tr_cheapest_pd.columns))\ndtrain_loyalty     = xgb.DMatrix(data_xgb_tr_loyalty_pd,     label=y_tr_loyalty_pd,     group=group_sizes_tr_loyalty,     feature_names=list(data_xgb_tr_loyalty_pd.columns))\n#dtrain_biz         = xgb.DMatrix(data_xgb_tr_biz_pd,         label=y_tr_biz_pd,         group=group_sizes_tr_biz,         feature_names=list(data_xgb_tr_biz_pd.columns))\n\n# Validation and Test remain unchanged\ndval = xgb.DMatrix(data_xgb_va.drop('ranker_id').to_pandas(), label=y_va['selected'].to_numpy(), group=group_sizes_va, feature_names=data_xgb.drop('ranker_id').columns)\ndtest = xgb.DMatrix(data_xgb_te.drop('ranker_id').to_pandas(), label=y_te['selected'].to_numpy(), group=group_sizes_te, feature_names=data_xgb.drop('ranker_id').columns)\n\n# ----------------- Train XGBoost Models -----------------\nfor portion, params in xgb_params.items():\n    print(f\"\\nTraining XGBoost model with portion: {portion}...\")\n\n    train_map = {\n        'full':        dtrain_full,\n        'small':       dtrain_small,\n        'big':         dtrain_big,\n        'min0':        dtrain_min_0,\n        #'min1':        dtrain_min_1,\n        'one_way':     dtrain_one_way,\n        'popular_route': dtrain_popular_route,\n        #'last_minute': dtrain_last_minute,\n        #'cheapest':    dtrain_cheapest,\n        #'direct':      dtrain_direct,\n        'loyalty':     dtrain_loyalty,\n        #'biz':         dtrain_biz\n    }\n\n    train = train_map[portion]\n    val   = dval\n\n    params = params.copy()\n    params['seed'] = RANDOM_STATE\n\n    model = xgb.train(\n        params,\n        train,\n        num_boost_round=860,\n        evals=[(train, 'train'), (val, 'val')],\n        # early_stopping_rounds=100,\n        verbose_eval=50\n    )\n\n    all_models.append(('xgb', RANDOM_STATE, portion, model))\n\n    # Feature importance\n    xgb_fi = pd.DataFrame.from_dict(\n        model.get_score(importance_type='gain'),\n        orient='index',\n        columns=['importance_gain']\n    ).reset_index()\n    xgb_fi.columns = ['feature', 'importance_gain']\n    xgb_fi['importance_split'] = list(model.get_score(importance_type='weight').values())\n    xgb_fi['seed'] = RANDOM_STATE\n    xgb_fi['model_type'] = 'xgb'\n    xgb_fi['portion'] = portion\n    feature_importances.append(xgb_fi)\n\n# ----------------- Optional: Train LightGBM -----------------\n# Uncomment if needed\n\n# for seed in lgb_seeds:\n#     print(f\"\\nTraining LightGBM model with seed {seed}...\")\n#     lgb_train = lgb.Dataset(\n#         data=data_xgb_tr.drop('ranker_id').to_pandas(),\n#         label=y_tr['selected'].to_numpy(),\n#         group=group_sizes_tr_lgb,\n#         feature_name=data_xgb.drop('ranker_id').columns,\n#         free_raw_data=False\n#     )\n#     lgb_val = lgb.Dataset(\n#         data=data_xgb_va.drop('ranker_id').to_pandas(),\n#         label=y_va['selected'].to_numpy(),\n#         group=group_sizes_va_lgb,\n#         feature_name=data_xgb.drop('ranker_id').columns,\n#         reference=lgb_train,\n#         free_raw_data=False\n#     )\n#     params = lgb_rank_params.copy()\n#     params['seed'] = seed\n\n#     model = lgb.train(\n#         params,\n#         lgb_train,\n#         num_boost_round=1700,\n#         valid_sets=[lgb_train, lgb_val],\n#         callbacks=[lgb.early_stopping(300), lgb.log_evaluation(50)]\n#     )\n\n#     all_models.append(('lgb', seed, model))\n\n#     fi_df = pd.DataFrame({\n#         'feature': data_xgb.drop('ranker_id').columns,\n#         'importance_split': model.feature_importance(importance_type='split'),\n#         'importance_gain': model.feature_importance(importance_type='gain'),\n#         'seed': seed,\n#         'model_type': 'lgb'\n#     })\n#     feature_importances.append(fi_df)\n\n# ----------------- Combine Feature Importances -----------------\nall_feature_importance = pd.concat(feature_importances, ignore_index=True)\n\nprint(\"\\n✅ Training completed. Total models:\", len(all_models))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T10:48:20.382572Z","iopub.execute_input":"2025-08-15T10:48:20.382812Z"}},"outputs":[{"name":"stdout","text":"\nTraining XGBoost model with portion: full...\n[0]\ttrain-ndcg@3:0.34421\tval-ndcg@3:0.37796\n[50]\ttrain-ndcg@3:0.59635\tval-ndcg@3:0.64229\n[100]\ttrain-ndcg@3:0.63850\tval-ndcg@3:0.68166\n[150]\ttrain-ndcg@3:0.66857\tval-ndcg@3:0.70839\n[200]\ttrain-ndcg@3:0.69526\tval-ndcg@3:0.73210\n[250]\ttrain-ndcg@3:0.71705\tval-ndcg@3:0.74901\n[300]\ttrain-ndcg@3:0.73691\tval-ndcg@3:0.76592\n[350]\ttrain-ndcg@3:0.75579\tval-ndcg@3:0.78253\n[400]\ttrain-ndcg@3:0.77286\tval-ndcg@3:0.79854\n[450]\ttrain-ndcg@3:0.78856\tval-ndcg@3:0.81141\n[500]\ttrain-ndcg@3:0.80170\tval-ndcg@3:0.82480\n[550]\ttrain-ndcg@3:0.81450\tval-ndcg@3:0.83574\n[600]\ttrain-ndcg@3:0.82590\tval-ndcg@3:0.84660\n[650]\ttrain-ndcg@3:0.83549\tval-ndcg@3:0.85518\n[700]\ttrain-ndcg@3:0.84266\tval-ndcg@3:0.86130\n[750]\ttrain-ndcg@3:0.84954\tval-ndcg@3:0.86732\n[800]\ttrain-ndcg@3:0.85517\tval-ndcg@3:0.87288\n[850]\ttrain-ndcg@3:0.85946\tval-ndcg@3:0.87604\n[859]\ttrain-ndcg@3:0.86053\tval-ndcg@3:0.87687\n\nTraining XGBoost model with portion: small...\n[0]\ttrain-ndcg@3:0.54588\tval-ndcg@3:0.36050\n[50]\ttrain-ndcg@3:0.83685\tval-ndcg@3:0.56069\n[100]\ttrain-ndcg@3:0.87272\tval-ndcg@3:0.58151\n[150]\ttrain-ndcg@3:0.88987\tval-ndcg@3:0.59277\n[200]\ttrain-ndcg@3:0.90176\tval-ndcg@3:0.60052\n[250]\ttrain-ndcg@3:0.91017\tval-ndcg@3:0.60473\n[300]\ttrain-ndcg@3:0.91694\tval-ndcg@3:0.60938\n[350]\ttrain-ndcg@3:0.92154\tval-ndcg@3:0.61238\n[400]\ttrain-ndcg@3:0.92484\tval-ndcg@3:0.61501\n[450]\ttrain-ndcg@3:0.92678\tval-ndcg@3:0.61677\n[500]\ttrain-ndcg@3:0.92872\tval-ndcg@3:0.61847\n[550]\ttrain-ndcg@3:0.93005\tval-ndcg@3:0.61932\n[600]\ttrain-ndcg@3:0.93103\tval-ndcg@3:0.61951\n[650]\ttrain-ndcg@3:0.93168\tval-ndcg@3:0.62062\n[700]\ttrain-ndcg@3:0.93257\tval-ndcg@3:0.62124\n[750]\ttrain-ndcg@3:0.93301\tval-ndcg@3:0.62162\n[800]\ttrain-ndcg@3:0.93338\tval-ndcg@3:0.62266\n[850]\ttrain-ndcg@3:0.93380\tval-ndcg@3:0.62302\n[859]\ttrain-ndcg@3:0.93379\tval-ndcg@3:0.62298\n\nTraining XGBoost model with portion: big...\n[0]\ttrain-ndcg@3:0.15905\tval-ndcg@3:0.35493\n[50]\ttrain-ndcg@3:0.48766\tval-ndcg@3:0.57392\n[100]\ttrain-ndcg@3:0.55207\tval-ndcg@3:0.60488\n[150]\ttrain-ndcg@3:0.59288\tval-ndcg@3:0.62179\n[200]\ttrain-ndcg@3:0.62761\tval-ndcg@3:0.63790\n[250]\ttrain-ndcg@3:0.65894\tval-ndcg@3:0.65247\n[300]\ttrain-ndcg@3:0.69222\tval-ndcg@3:0.66996\n[350]\ttrain-ndcg@3:0.72325\tval-ndcg@3:0.68460\n[400]\ttrain-ndcg@3:0.75062\tval-ndcg@3:0.69746\n[450]\ttrain-ndcg@3:0.77310\tval-ndcg@3:0.70893\n[500]\ttrain-ndcg@3:0.79431\tval-ndcg@3:0.71906\n[550]\ttrain-ndcg@3:0.81038\tval-ndcg@3:0.72461\n[600]\ttrain-ndcg@3:0.82317\tval-ndcg@3:0.72993\n[650]\ttrain-ndcg@3:0.83334\tval-ndcg@3:0.73595\n[700]\ttrain-ndcg@3:0.84290\tval-ndcg@3:0.74005\n[750]\ttrain-ndcg@3:0.84978\tval-ndcg@3:0.74297\n[800]\ttrain-ndcg@3:0.85513\tval-ndcg@3:0.74458\n[850]\ttrain-ndcg@3:0.85967\tval-ndcg@3:0.74712\n[859]\ttrain-ndcg@3:0.86030\tval-ndcg@3:0.74737\n\nTraining XGBoost model with portion: min0...\n[0]\ttrain-ndcg@3:0.31526\tval-ndcg@3:0.36736\n[50]\ttrain-ndcg@3:0.60659\tval-ndcg@3:0.60657\n[100]\ttrain-ndcg@3:0.65494\tval-ndcg@3:0.63620\n[150]\ttrain-ndcg@3:0.69064\tval-ndcg@3:0.66116\n[200]\ttrain-ndcg@3:0.71866\tval-ndcg@3:0.67728\n[250]\ttrain-ndcg@3:0.75234\tval-ndcg@3:0.70064\n[300]\ttrain-ndcg@3:0.77671\tval-ndcg@3:0.71762","output_type":"stream"}],"execution_count":null},{"cell_type":"code","source":"# Group by feature and model_type\nagg_importance = (\n    all_feature_importance\n    .groupby([\"model_type\", \"feature\"])[[\"importance_split\", \"importance_gain\"]]\n    .mean()\n    .reset_index()\n)\n\n# Separate and sort LightGBM and XGBoost\n# lgb_fi_sorted = (\n#     agg_importance[agg_importance[\"model_type\"] == \"lgb\"]\n#     .sort_values(\"importance_gain\", ascending=False)\n# )\n\nxgb_fi_sorted = (\n    agg_importance[agg_importance[\"model_type\"] == \"xgb\"]\n    .sort_values(\"importance_gain\", ascending=False)\n)\n\n# # Display top features\n# print(\"🔝 Top LightGBM Features (by Gain):\")\n# print(lgb_fi_sorted.head(30))\n\npd.set_option('display.max_rows', None)     # Show all rows\npd.set_option('display.max_columns', None)  # Show all columns\npd.set_option('display.width', 0)           # Auto-detect width\npd.set_option('display.max_colwidth', None) # Show full content in cells\n\nprint(xgb_fi_sorted.head(100))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import ndcg_score\nimport numpy as np\n\n# --- Separate LGB and XGB models from all_models ---\n#lgb_models = [m for (typ, _, m) in all_models if typ == 'lgb']\nxgb_models = [model for (typ, _, _, model) in all_models if typ == 'xgb']\n\n# --- Predict on validation set and average ---\nprint(\"\\n📊 Averaging predictions on validation set...\")\n\n# LightGBM predictions\n#preds_val_lgb = np.mean([model.predict(data_xgb_va) for model in lgb_models], axis=0)\n\n# XGBoost predictions\ndata_xgb_va_pd = data_xgb_va.drop('ranker_id').to_pandas()\ndval = xgb.DMatrix(data_xgb_va_pd)\npreds_val_xgb = np.mean([model.predict(dval) for model in xgb_models], axis=0)\n\n# Combine XGBoost predictions\npreds_val = preds_val_xgb\n\n# Convert labels to numeric if needed\ny_true = y_va['selected'].to_numpy().flatten()\n\n# Evaluate NDCG@3\nndcg_val = ndcg_score([y_true], [preds_val], k=3)\nprint(f\"✅ Ensemble NDCG@3: {ndcg_val:.4f}\")\n\n# Evaluate HitRate@3 (if function is defined)\nensemble_hr3 = hitrate_at_3(y_true, preds_val, groups_va['ranker_id'])\nprint(f\"🎯 Ensemble HitRate@3: {ensemble_hr3:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, penalty_factor=0.12):\n    COLS_TO_COMPARE = [\n        \"legs0_departureAt\",\n        \"legs0_arrivalAt\",\n        \"legs1_departureAt\",\n        \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\",\n        \"legs1_segments0_flightNumber\",\n        \"legs0_segments0_aircraft_code\",\n        \"legs1_segments0_aircraft_code\",\n        \"legs0_segments0_departureFrom_airport_iata\",\n        \"legs1_segments0_departureFrom_airport_iata\",\n    ]\n\n    test = test.with_columns(\n        [pl.col(c).cast(str).fill_null(\"NULL\") for c in COLS_TO_COMPARE]\n    )\n\n    df = submission_xgb.join(test, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    df = df.with_columns(\n        (\n            pl.col(\"legs0_departureAt\")\n            + \"_\"\n            + pl.col(\"legs0_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs1_departureAt\")\n            + \"_\"\n            + pl.col(\"legs1_arrivalAt\")\n            + \"_\"\n            + pl.col(\"legs0_segments0_flightNumber\")\n            + \"_\"\n            + pl.col(\"legs1_segments0_flightNumber\")\n        ).alias(\"flight_hash\")\n    )\n\n    df = df.with_columns(\n        pl.max(\"pred_score\")\n        .over([\"ranker_id\", \"flight_hash\"])\n        .alias(\"max_score_same_flight\")\n    )\n\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\")\n            - penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\"))\n        ).alias(\"reorder_score\")\n    )\n\n    df = df.with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Predict on test set ---\nprint(\"\\n📦 Generating predictions on test set...\")\n\n# LightGBM test predictions\n#preds_test_lgb = np.mean([model.predict(data_xgb_te) for model in lgb_models], axis=0)\n\n# # XGBoost test predictions\n\ndata_xgb_te_pd = data_xgb_te.drop('ranker_id').to_pandas()\ndtest = xgb.DMatrix(data_xgb_te_pd)\npreds_test_xgb = np.mean([model.predict(dtest) for model in xgb_models], axis=0)\n\n# Combined test ensemble\nensemble_test_preds =  preds_test_xgb\n\nsubmission_df = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', ensemble_test_preds))\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n)\n\ntop = re_rank(test, submission_df)\n\nsubmission_df = (\n    submission_df.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n    .with_columns(\n        [\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ]\n    )\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\n\n# --- Save to CSV ---\nsubmission_df.write_csv('submission.csv')\nprint(\"\\n✅ Submission file 'submission.csv' created successfully.\")\nprint(submission_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import xgboost as xgb\n\n# xgb_rank_params = {\n#     'objective': 'rank:ndcg',       # or use 'rank:pairwise' as alternative\n#     'learning_rate': 0.1,\n#     'gamma': 1.0,\n#     'min_child_weight': 30,\n#     'max_depth': 6,\n#     'subsample': 0.85,\n#     'colsample_bytree': 0.6,\n#     'lambda': 1.0,\n#     'alpha': 0.1,\n#     'eval_metric': 'ndcg@3',\n#     'verbosity': 1,\n#     'seed': 42,\n#     'tree_method': 'hist',          # optional for speed\n# }\n\n# print(\"Training XGBoost ranker...\")\n\n# # Create DMatrix for XGBoost\n# xgb_train = xgb.DMatrix(data_xgb_tr, label=y_tr.to_numpy().flatten())\n# xgb_val = xgb.DMatrix(data_xgb_va, label=y_va.to_numpy().flatten())\n\n# xgb_train.set_group(group_sizes_tr)\n# xgb_val.set_group(group_sizes_va)\n\n# # Train XGBoost ranking model\n# xgb_model = xgb.train(\n#     xgb_rank_params,\n#     dtrain=xgb_train,\n#     num_boost_round=500,\n#     evals=[(xgb_train, \"train\"), (xgb_val, \"valid\")],\n#     early_stopping_rounds=50,\n#     verbose_eval=50\n# )\n\n# xgb_fi = pd.DataFrame.from_dict(xgb_model.get_score(importance_type='gain'), orient='index', columns=['importance_gain']).reset_index()\n# xgb_fi.columns = ['feature', 'importance_gain']\n# xgb_fi = xgb_fi.sort_values('importance_gain', ascending=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# xgb_fi.head(30)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# import numpy as np\n# from sklearn.metrics import ndcg_score\n\n# lgb_rank_params = {\n#     'objective': 'lambdarank',\n#     'metric': 'ndcg',\n#     'boosting_type': 'gbdt',\n#     'eval_at': [3],\n#     'num_leaves': 137,\n#     'learning_rate': 0.1923609,\n#     'min_child_samples': 69,\n#     'lambda_l1': 0.0017863,\n#     'lambda_l2': 7.8818,\n#     'feature_fraction': 0.6015,\n#     'bagging_fraction': 0.8536,\n#     'bagging_freq': 7,\n#     'verbosity': -1,\n#     'label_gain': [0, 1]\n# }\n\n\n# models = []\n# lgb_seeds = [12, 32, 42]\n\n# feature_importances = []\n\n# for seed in lgb_seeds:\n#     params = lgb_rank_params.copy()\n#     params['seed'] = seed\n\n#     print(f\"Training LightGBM model with seed {seed}...\")\n    \n#     lgb_train = lgb.Dataset(\n#         data=data_xgb_tr,\n#         label=y_tr.to_numpy().flatten(),\n#         group=group_sizes_tr,\n#         feature_name=feature_cols,\n#         free_raw_data=False\n#     )\n\n#     lgb_val = lgb.Dataset(\n#         data=data_xgb_va,\n#         label=y_va.to_numpy().flatten(),\n#         group=group_sizes_va,\n#         feature_name=feature_cols,\n#         reference=lgb_train,\n#         free_raw_data=False\n#     )\n\n#     model = lgb.train(\n#         params,\n#         lgb_train,\n#         num_boost_round=1500,\n#         valid_sets=[lgb_train, lgb_val],\n#         callbacks=[lgb.early_stopping(100), lgb.log_evaluation(50)]\n#     )\n\n#     models.append(model)\n\n#     # Store feature importance\n#     fi_df = pd.DataFrame({\n#         'feature': feature_cols,\n#         'importance_split': model.feature_importance(importance_type='split'),\n#         'importance_gain': model.feature_importance(importance_type='gain'),\n#         'seed': seed\n#     })\n#     feature_importances.append(fi_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Combine all importance dataframes\n# all_fi_df = pd.concat(feature_importances)\n\n# # Average importance across seeds\n# avg_fi = all_fi_df.groupby(\"feature\")[[\"importance_split\", \"importance_gain\"]].mean().sort_values(\"importance_gain\", ascending=False)\n\n# # Show top 20 features\n# print(avg_fi.head(30))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# avg_fi['norm_gain'] = 100 * avg_fi['importance_gain'] / avg_fi['importance_gain'].sum()\n# avg_fi['norm_gain'].head(20) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# avg_fi.head(30).sort_values(\"importance_gain\").plot(kind='barh', figsize=(15, 12))\n# plt.title(\"Top 30 Features by Gain\")\n# plt.xlabel(\"Average Gain Importance\")\n# plt.tight_layout()\n# plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Predict on validation set and average\n# preds_val = np.mean([model.predict(data_xgb_va) for model in models], axis=0)\n\n# from sklearn.metrics import ndcg_score\n# # Evaluate using NDCG@3\n# ndcg_val = ndcg_score([y_va.to_numpy().flatten()], [preds_val], k=3)\n# print(f\"✅ Ensemble NDCG@3: {ndcg_val:.4f}\")\n\n# lgb_ensemble_hr3 = hitrate_at_3(y_va['selected'], preds_val, groups_va['ranker_id'])\n# print(f\"LGBM ensemble HitRate@3:   {lgb_ensemble_hr3:.4f}\")\n\n# lgb_ensemble_test_preds  = np.mean([model.predict(data_xgb_te) for model in models], axis=0)\n\n# submission_df = test.select(['Id', 'ranker_id']).with_columns(\n#     pl.Series(name=\"lgb_score\", values=lgb_ensemble_test_preds)\n# ).with_columns(\n#     # Rank predictions descending (best scores first) within each group\n#     pl.col(\"lgb_score\").rank(method=\"ordinal\", descending=True).over(\"ranker_id\").cast(pl.Int32).alias(\"selected\")\n# ).select([\"Id\", \"ranker_id\", \"selected\"])\n\n# # Save to CSV\n# submission_df.write_csv('submission.csv')\n\n# print(\"\\n✅ Submission file 'submission.csv' created successfully.\")\n# print(submission_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}