{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"super starrrr\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T12:11:21.595429Z","iopub.execute_input":"2025-08-16T12:11:21.595833Z","iopub.status.idle":"2025-08-16T12:11:21.608081Z","shell.execute_reply.started":"2025-08-16T12:11:21.595808Z","shell.execute_reply":"2025-08-16T12:11:21.603921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n!pip install -U xgboost\n!pip install -U polars\n\nimport polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nfrom typing import List, Tuple, Optional\nimport logging\n\n# Configure logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n# Constants\nRANDOM_STATE = 42\nTRAIN_VAL_SPLIT = 16487352\nPENALTY_FACTOR = 0.1\n\n# Set random seed\nnp.random.seed(RANDOM_STATE)\n\nclass FlightDataProcessor:\n    \"\"\"Encapsulate data processing logic for flight recommendation system.\"\"\"\n    \n    def __init__(self):\n        self.categorical_features = []\n        self.feature_columns = []\n        \n    def load_data(self, train_path: str, test_path: str) -> pl.DataFrame:\n        \"\"\"Load and combine train/test data.\"\"\"\n        logger.info(\"Loading data...\")\n        train = pl.read_parquet(train_path).drop('__index_level_0__')\n        test = (pl.read_parquet(test_path)\n                .drop('__index_level_0__')\n                .with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\")))\n        \n        return pl.concat((train, test))\n    \n    @staticmethod\n    def duration_to_minutes(col: pl.Expr) -> pl.Expr:\n        \"\"\"Convert duration string to minutes more efficiently.\"\"\"\n        # Extract days and time parts in one pass\n        days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n        time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(\n            col.str.replace(r\"^\\d+\\.\", \"\")\n        ).otherwise(col)\n        hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n        minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n        \n        return (days + hours + minutes).fill_null(0)\n    \n    def create_price_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create price-related features.\"\"\"\n        return df.with_columns([\n            (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n            pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        ])\n    \n    def create_duration_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create duration-related features.\"\"\"\n        return df.with_columns([\n            (pl.col(\"legs0_duration\").fill_null(0) + \n             pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n            pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n                .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n                .otherwise(1.0).alias(\"duration_ratio\"),\n        ])\n    \n    def create_trip_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create trip-type and routing features.\"\"\"\n        # Check for marketing carrier columns\n        mc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' \n                  for l in (0, 1) for s in range(4)]\n        mc_exists = [col for col in mc_cols if col in df.columns]\n        \n        return df.with_columns([\n            # Trip type\n            (pl.col(\"legs1_duration\").is_null() | \n             (pl.col(\"legs1_duration\") == 0) | \n             pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null())\n                .cast(pl.Int32).alias(\"is_one_way\"),\n            \n            # Segment counts\n            (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) \n                             for col in mc_exists) if mc_exists else pl.lit(0))\n                .alias(\"l0_seg\"),\n            \n            # Route popularity\n            pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \n                                       \"MOWLED\", \"LEDMOW\"])\n                .cast(pl.Int32).alias(\"is_popular_route\"),\n        ])\n    \n    def create_passenger_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create passenger-related features.\"\"\"\n        return df.with_columns([\n            # Frequent flyer\n            (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n             (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32))\n                .alias(\"n_ff_programs\"),\n            \n            # Binary features\n            pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32)\n                .alias(\"has_corporate_tariff\"),\n            (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32)\n                .alias(\"has_access_tp\"),\n            \n            # Cancellation/exchange rules\n            ((pl.col(\"miniRules0_monetaryAmount\") == 0) & \n             (pl.col(\"miniRules0_statusInfos\") == 1))\n                .cast(pl.Int8).alias(\"free_cancel\"),\n            ((pl.col(\"miniRules1_monetaryAmount\") == 0) & \n             (pl.col(\"miniRules1_statusInfos\") == 1))\n                .cast(pl.Int8).alias(\"free_exchange\"),\n        ])\n    \n    def create_cabin_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create cabin class features.\"\"\"\n        return df.with_columns([\n            pl.mean_horizontal([\"legs0_segments0_cabinClass\", \n                              \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n            (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n             pl.col(\"legs1_segments0_cabinClass\").fill_null(0))\n                .alias(\"cabin_class_diff\"),\n        ])\n    \n    def create_segment_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create segment-based features.\"\"\"\n        # Create segment counts\n        seg_exprs = []\n        for leg in (0, 1):\n            seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) \n                       if f\"legs{leg}_segments{s}_duration\" in df.columns]\n            if seg_cols:\n                seg_exprs.append(\n                    pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                        .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n                )\n            else:\n                seg_exprs.append(pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"))\n        \n        df = df.with_columns(seg_exprs)\n        \n        # Derived features\n        return df.with_columns([\n            (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n            (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n            pl.when(pl.col(\"is_one_way\") == 1).then(0)\n                .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32))\n                .alias(\"is_direct_leg1\"),\n        ]).with_columns([\n            (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\"))\n                .cast(pl.Int32).alias(\"both_direct\"),\n            ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0))\n                .cast(pl.Int32).alias(\"is_vip_freq\"),\n            pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n        ])\n    \n    def create_time_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create time-based features.\"\"\"\n        time_exprs = []\n        time_cols = [\"legs0_departureAt\", \"legs0_arrivalAt\", \n                    \"legs1_departureAt\", \"legs1_arrivalAt\"]\n        \n        for col in time_cols:\n            if col in df.columns:\n                dt = pl.col(col).str.to_datetime(strict=False)\n                h = dt.dt.hour().fill_null(12)\n                time_exprs.extend([\n                    h.alias(f\"{col}_hour\"),\n                    dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n                    (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20)))\n                        .cast(pl.Int32).alias(f\"{col}_business_time\")\n                ])\n        \n        return df.with_columns(time_exprs) if time_exprs else df\n    \n    def create_ranking_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Create ranking and competitive features.\"\"\"\n        df = df.with_columns([\n            pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n        ])\n        \n        # Basic ranks\n        rank_exprs = [\n            pl.col(\"totalPrice\").rank().over(\"ranker_id\").alias(\"price_rank\"),\n            pl.col(\"total_duration\").rank().over(\"ranker_id\").alias(\"duration_rank\"),\n        ]\n        \n        # Price-specific features\n        price_exprs = [\n            (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n             pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n            (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\"))\n                .cast(pl.Int32).alias(\"is_cheapest\"),\n            ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n             (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n            (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\"))\n                .cast(pl.Int32).alias(\"is_min_segments\"),\n        ]\n        \n        return df.with_columns(rank_exprs + price_exprs)\n    \n    def add_carrier_features(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Add carrier-related features.\"\"\"\n        # Major carrier flag\n        if \"legs0_segments0_marketingCarrier_code\" in df.columns:\n            df = df.with_columns(\n                pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\"])\n                    .cast(pl.Int32).alias(\"is_major_carrier\")\n            )\n        else:\n            df = df.with_columns(pl.lit(0).alias(\"is_major_carrier\"))\n        \n        return df\n    \n    def add_direct_cheapest_feature(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Add direct cheapest flight feature.\"\"\"\n        direct_cheapest = (\n            df.filter(pl.col(\"is_direct_leg0\") == 1)\n            .group_by(\"ranker_id\")\n            .agg(pl.col(\"totalPrice\").min().alias(\"min_direct\"))\n        )\n        \n        return (df.join(direct_cheapest, on=\"ranker_id\", how=\"left\")\n                .with_columns(\n                    ((pl.col(\"is_direct_leg0\") == 1) & \n                     (pl.col(\"totalPrice\") == pl.col(\"min_direct\")))\n                        .cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\")\n                ).drop(\"min_direct\"))\n    \n    def add_popularity_features(self, df: pl.DataFrame, train: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Add carrier popularity features based on training data.\"\"\"\n        carrier0_pop = train.group_by('legs0_segments0_marketingCarrier_code').agg(\n            pl.mean('selected').alias('carrier0_pop')\n        )\n        carrier1_pop = train.group_by('legs1_segments0_marketingCarrier_code').agg(\n            pl.mean('selected').alias('carrier1_pop')\n        )\n        \n        return (df.join(carrier0_pop, on='legs0_segments0_marketingCarrier_code', how='left')\n                .join(carrier1_pop, on='legs1_segments0_marketingCarrier_code', how='left')\n                .with_columns([\n                    pl.col('carrier0_pop').fill_null(0.0),\n                    pl.col('carrier1_pop').fill_null(0.0),\n                ])\n                .with_columns([\n                    (pl.col('carrier0_pop') * pl.col('carrier1_pop'))\n                        .alias('carrier_pop_product'),\n                ]))\n    \n    def process_durations(self, df: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"Process duration columns.\"\"\"\n        dur_cols = [\"legs0_duration\", \"legs1_duration\"] + [\n            f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)\n        ]\n        dur_exprs = [self.duration_to_minutes(pl.col(c)).alias(c) \n                    for c in dur_cols if c in df.columns]\n        \n        return df.with_columns(dur_exprs) if dur_exprs else df\n    \n    def get_categorical_features(self) -> List[str]:\n        \"\"\"Define categorical features.\"\"\"\n        return [\n            'nationality', 'searchRoute', 'corporateTariffCode',\n            'bySelf', 'sex', 'companyID',\n            # Leg 0 segments\n            'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n            'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n            'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n            'legs0_segments0_flightNumber',\n            'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n            'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n            'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n            'legs0_segments1_flightNumber',\n            # Leg 1 segments\n            'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n            'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n            'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n            'legs1_segments0_flightNumber',\n            'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n            'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n            'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n            'legs1_segments1_flightNumber',\n        ]\n    \n    def get_exclude_columns(self) -> List[str]:\n        \"\"\"Define columns to exclude from features.\"\"\"\n        exclude_cols = [\n            'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n            'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n            'miniRules0_percentage', 'miniRules1_percentage',  # >90% missing\n            'frequentFlyer',  # Already processed\n            'pricingInfo_passengerCount'  # Constant\n        ]\n        \n        # Add high-missing segments\n        for leg in [0, 1]:\n            for seg in [0, 1]:\n                if seg == 0:\n                    suffixes = [\"seatsAvailable\"]\n                else:\n                    suffixes = [\n                        \"cabinClass\", \"seatsAvailable\", \"baggageAllowance_quantity\",\n                        \"baggageAllowance_weightMeasurementType\", \"aircraft_code\",\n                        \"arrivalTo_airport_city_iata\", \"arrivalTo_airport_iata\",\n                        \"departureFrom_airport_iata\", \"flightNumber\",\n                        \"marketingCarrier_code\", \"operatingCarrier_code\",\n                    ]\n                for suffix in suffixes:\n                    exclude_cols.append(f\"legs{leg}_segments{seg}_{suffix}\")\n        \n        # Exclude segment 2-3 columns (>98% missing)\n        for leg in [0, 1]:\n            for seg in [2, 3]:\n                for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', \n                             'arrivalTo_airport_iata', 'baggageAllowance_quantity',\n                             'baggageAllowance_weightMeasurementType', 'cabinClass',\n                             'departureFrom_airport_iata', 'duration', 'flightNumber',\n                             'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n                    exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n        \n        return exclude_cols\n    \n    def process_features(self, df: pl.DataFrame, train: pl.DataFrame) -> Tuple[pl.DataFrame, List[str]]:\n        \"\"\"Main feature processing pipeline.\"\"\"\n        logger.info(\"Processing durations...\")\n        df = self.process_durations(df)\n        \n        logger.info(\"Creating features...\")\n        df = self.create_price_features(df)\n        df = self.create_duration_features(df)\n        df = self.create_trip_features(df)\n        df = self.create_passenger_features(df)\n        df = self.create_cabin_features(df)\n        df = self.create_segment_features(df)\n        df = self.create_time_features(df)\n        df = self.create_ranking_features(df)\n        df = self.add_carrier_features(df)\n        df = self.add_direct_cheapest_feature(df)\n        df = self.add_popularity_features(df, train)\n        \n        # Fill nulls\n        logger.info(\"Filling nulls...\")\n        df = df.with_columns(\n            [pl.col(c).fill_null(0) for c in df.select(pl.selectors.numeric()).columns] +\n            [pl.col(c).fill_null(\"missing\") for c in df.select(pl.selectors.string()).columns]\n        )\n        \n        # Get feature columns\n        cat_features = self.get_categorical_features()\n        exclude_cols = self.get_exclude_columns()\n        feature_cols = [col for col in df.columns if col not in exclude_cols]\n        cat_features_final = [col for col in cat_features if col in feature_cols]\n        \n        logger.info(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n        \n        self.categorical_features = cat_features_final\n        self.feature_columns = feature_cols\n        \n        return df, feature_cols\n\ndef hitrate_at_3(y_true: np.ndarray, y_pred: np.ndarray, groups: np.ndarray) -> float:\n    \"\"\"Calculate hit rate at 3.\"\"\"\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )\n\ndef re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, \n           penalty_factor: float = PENALTY_FACTOR) -> pl.DataFrame:\n    \"\"\"Re-rank submissions to avoid duplicate flights.\"\"\"\n    COLS_TO_COMPARE = [\n        \"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\", \"legs1_segments0_flightNumber\",\n        \"legs0_segments0_aircraft_code\", \"legs1_segments0_aircraft_code\",\n        \"legs0_segments0_departureFrom_airport_iata\", \"legs1_segments0_departureFrom_airport_iata\",\n    ]\n\n    test_processed = test.with_columns(\n        [pl.col(c).cast(str).fill_null(\"NULL\") for c in COLS_TO_COMPARE]\n    )\n\n    df = submission_xgb.join(test_processed, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    # Create flight hash\n    df = df.with_columns(\n        pl.concat_str([pl.col(c) for c in COLS_TO_COMPARE], separator=\"_\")\n            .alias(\"flight_hash\")\n    )\n\n    # Apply penalty for duplicate flights\n    df = df.with_columns(\n        pl.max(\"pred_score\").over([\"ranker_id\", \"flight_hash\"])\n            .alias(\"max_score_same_flight\")\n    ).with_columns(\n        (pl.col(\"pred_score\") - \n         penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\")))\n            .alias(\"reorder_score\")\n    ).with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])\n\ndef main():\n    \"\"\"Main execution function.\"\"\"\n    # Initialize processor\n    processor = FlightDataProcessor()\n    \n    # Load data\n    data_raw = processor.load_data(\n        '/kaggle/input/aeroclub-recsys-2025/train.parquet',\n        '/kaggle/input/aeroclub-recsys-2025/test.parquet'\n    )\n    \n    # Load train separately for popularity features\n    train = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\n    test = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__')\n    \n    # Process features\n    data, feature_cols = processor.process_features(data_raw, train)\n    \n    # Prepare data for XGBoost\n    X = data.select(feature_cols)\n    y = data.select('selected')\n    groups = data.select('ranker_id')\n    \n    # Encode categorical features\n    data_xgb = X.with_columns([\n        (pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int16) \n        for c in processor.categorical_features\n    ])\n    \n    # Split data\n    n2 = train.height\n    data_xgb_tr = data_xgb[:TRAIN_VAL_SPLIT]\n    data_xgb_va = data_xgb[TRAIN_VAL_SPLIT:n2]\n    data_xgb_te = data_xgb[n2:]\n    \n    y_tr, y_va, y_te = y[:TRAIN_VAL_SPLIT], y[TRAIN_VAL_SPLIT:n2], y[n2:]\n    groups_tr, groups_va, groups_te = groups[:TRAIN_VAL_SPLIT], groups[TRAIN_VAL_SPLIT:n2], groups[n2:]\n    \n    # Prepare group sizes for XGBoost\n    group_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n    group_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n    group_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n    \n    # Create DMatrix objects\n    dtrain = xgb.DMatrix(data_xgb_tr, label=y_tr, group=group_sizes_tr, feature_names=data_xgb.columns)\n    dval = xgb.DMatrix(data_xgb_va, label=y_va, group=group_sizes_va, feature_names=data_xgb.columns)\n    dtest = xgb.DMatrix(data_xgb_te, label=y_te, group=group_sizes_te, feature_names=data_xgb.columns)\n\n    print(\"nodel training is finally starting\")\n    # XGBoost parameters\n    xgb_params = {\n        'objective': 'rank:pairwise',\n        'eval_metric': 'ndcg@3',\n        \"learning_rate\": 0.022641389657079056,\n        \"max_depth\": 14,\n        \"min_child_weight\": 2,\n        \"subsample\": 0.8842234913702768,\n        \"colsample_bytree\": 0.45840689146263086,\n        \"gamma\": 3.3084297630544888,\n        \"lambda\": 6.952586917313028,\n        \"alpha\": 0.6395254133055179,\n        'seed': RANDOM_STATE,\n        'n_jobs': -1,\n        'verbosity': 1\n    }\n    \n    # Train model\n    logger.info(\"Training XGBoost model...\")\n    xgb_model = xgb.train(\n        xgb_params,\n        dtrain,\n        num_boost_round=800,\n        evals=[(dtrain, 'train'), (dval, 'val')],\n        verbose_eval=50\n    )\n    \n    # Generate predictions\n    logger.info(\"Generating predictions...\")\n    submission_xgb = (\n        test.select(['Id', 'ranker_id'])\n        .with_columns(pl.Series('pred_score', xgb_model.predict(dtest)))\n        .with_columns(\n            pl.col('pred_score')\n            .rank(method='ordinal', descending=True)\n            .over('ranker_id')\n            .cast(pl.Int32)\n            .alias('selected')\n        )\n        .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n    )\n    \n    # Apply re-ranking\n    logger.info(\"Applying re-ranking...\")\n    top = re_rank(test, submission_xgb)\n    submission_final = (\n        submission_xgb.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n        .with_columns([\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ])\n        .select([\"Id\", \"ranker_id\", \"selected\"])\n    )\n    \n    # Save submission\n    logger.info(\"Saving submission...\")\n    submission_final.write_csv('submission.csv')\n    logger.info(\"Pipeline completed successfully!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T13:33:42.184802Z","iopub.execute_input":"2025-08-16T13:33:42.185214Z","iopub.status.idle":"2025-08-16T13:40:37.059253Z","shell.execute_reply.started":"2025-08-16T13:33:42.185184Z","shell.execute_reply":"2025-08-16T13:40:37.053960Z"}},"outputs":[],"execution_count":null}]}