{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.18","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"},{"sourceId":12777232,"sourceType":"datasetVersion","datasetId":8077690}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install xgboost\n!pip install polars\n!pip install optuna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T08:23:42.231334Z","iopub.execute_input":"2025-08-15T08:23:42.231545Z","iopub.status.idle":"2025-08-15T08:24:06.791134Z","shell.execute_reply.started":"2025-08-15T08:23:42.231522Z","shell.execute_reply":"2025-08-15T08:24:06.784999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T08:27:33.621015Z","iopub.execute_input":"2025-08-15T08:27:33.621378Z","iopub.status.idle":"2025-08-15T08:27:37.218874Z","shell.execute_reply.started":"2025-08-15T08:27:33.621343Z","shell.execute_reply":"2025-08-15T08:27:37.213191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport xgboost as xgb\nfrom sklearn.preprocessing import QuantileTransformer\nimport optuna\nfrom sklearn.metrics import ndcg_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T08:27:39.484819Z","iopub.execute_input":"2025-08-15T08:27:39.485178Z","iopub.status.idle":"2025-08-15T08:27:43.822706Z","shell.execute_reply.started":"2025-08-15T08:27:39.485148Z","shell.execute_reply":"2025-08-15T08:27:43.818134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/train.parquet').drop('__index_level_0__')\ntest = pl.read_parquet('/kaggle/input/aeroclub-recsys-2025/test.parquet').drop('__index_level_0__').with_columns(pl.lit(0, dtype=pl.Int64).alias(\"selected\"))\n\ndata_raw = pl.concat((train, test))\nprint(f\"Total data shape: {data_raw.shape}\")\nprint(f\"Train shape: {train.shape}, Test shape: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:13:39.452504Z","iopub.execute_input":"2025-08-16T05:13:39.452846Z","iopub.status.idle":"2025-08-16T05:13:42.888683Z","shell.execute_reply.started":"2025-08-16T05:13:39.452818Z","shell.execute_reply":"2025-08-16T05:13:42.883727Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Enhanced Helpers","metadata":{}},{"cell_type":"code","source":"def hitrate_at_3(y_true, y_pred, groups):\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(3)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )\n\ndef hitrate_at_k(y_true, y_pred, groups, k):\n    \"\"\"Calculate hitrate@k for any k\"\"\"\n    df = pl.DataFrame({\n        'group': groups,\n        'pred': y_pred,\n        'true': y_true\n    })\n    \n    return (\n        df.filter(pl.col(\"group\").count().over(\"group\") > 10)\n        .sort([\"group\", \"pred\"], descending=[False, True])\n        .group_by(\"group\", maintain_order=True)\n        .head(k)\n        .group_by(\"group\")\n        .agg(pl.col(\"true\").max())\n        .select(pl.col(\"true\").mean())\n        .item()\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:13:47.859202Z","iopub.execute_input":"2025-08-16T05:13:47.859544Z","iopub.status.idle":"2025-08-16T05:13:47.874108Z","shell.execute_reply.started":"2025-08-16T05:13:47.859517Z","shell.execute_reply":"2025-08-16T05:13:47.867698Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Enhanced Feature Engineering","metadata":{}},{"cell_type":"code","source":"df = data_raw.clone()\n\n# More efficient duration to minutes converter\ndef dur_to_min(col):\n    # Extract days and time parts in one pass\n    days = col.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Int64).fill_null(0) * 1440\n    time_str = pl.when(col.str.contains(r\"^\\d+\\.\")).then(col.str.replace(r\"^\\d+\\.\", \"\")).otherwise(col)\n    hours = time_str.str.extract(r\"^(\\d+):\", 1).cast(pl.Int64).fill_null(0) * 60\n    minutes = time_str.str.extract(r\":(\\d+):\", 1).cast(pl.Int64).fill_null(0)\n    return (days + hours + minutes).fill_null(0)\n\n# Process duration columns\ndur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1)]\ndur_exprs = [dur_to_min(pl.col(c)).alias(c) for c in dur_cols if c in df.columns]\n\n# Apply duration transformations first\nif dur_exprs:\n    df = df.with_columns(dur_exprs)\n\nprint(\"Duration columns processed...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:13:49.868046Z","iopub.execute_input":"2025-08-16T05:13:49.868437Z","iopub.status.idle":"2025-08-16T05:14:07.662307Z","shell.execute_reply.started":"2025-08-16T05:13:49.868406Z","shell.execute_reply":"2025-08-16T05:14:07.657696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate key columns and add fallbacks\nrequired_cols = ['totalPrice', 'taxes', 'ranker_id', 'selected']\nmissing_required = [col for col in required_cols if col not in df.columns]\nif missing_required:\n    print(f\"Missing required columns: {missing_required}\")\n    raise ValueError(f\"Required columns missing: {missing_required}\")\n\n# Add fallback columns if they don't exist\nfallback_columns = []\nif 'legs0_duration' not in df.columns:\n    fallback_columns.append(pl.lit(0).alias('legs0_duration'))\nif 'legs1_duration' not in df.columns:\n    fallback_columns.append(pl.lit(0).alias('legs1_duration'))\nif 'pricingInfo_passengerCount' not in df.columns:\n    fallback_columns.append(pl.lit(1).alias('pricingInfo_passengerCount'))\n\nif fallback_columns:\n    df = df.with_columns(fallback_columns)\n    print(f\"Added {len(fallback_columns)} fallback columns\")\n\n# Precompute marketing carrier columns check\nmc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\nmc_exists = [col for col in mc_cols if col in df.columns]\nprint(f\"Found {len(mc_exists)} marketing carrier columns\")\n\n# STEP 1: ORIGINAL BASIC FEATURES - Enhanced with additional features (no self-references)\ndf = df.with_columns([\n        # === ORIGINAL PRICE FEATURES ===\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") / (pl.col(\"totalPrice\") + 1)).alias(\"tax_rate\"),\n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        \n        # === NEW ENHANCED PRICE FEATURES ===\n        pl.col(\"totalPrice\").sqrt().alias(\"sqrt_price\"),\n        (pl.col(\"totalPrice\") ** 0.25).alias(\"fourth_root_price\"),\n        (pl.col(\"totalPrice\") / pl.col(\"pricingInfo_passengerCount\").clip(lower_bound=1)).alias(\"price_per_passenger\"),\n        (pl.col(\"taxes\").log1p()).alias(\"log_taxes\"),\n        (pl.col(\"totalPrice\") - pl.col(\"taxes\")).clip(lower_bound=0).alias(\"base_fare\"),\n        \n        # === ORIGINAL DURATION FEATURES ===\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 1))\n            .otherwise(1.0).alias(\"duration_ratio\"),\n            \n        # === NEW ENHANCED DURATION FEATURES ===\n        (pl.col(\"legs0_duration\").fill_null(0)).log1p().alias(\"log_leg0_duration\"),\n        (pl.col(\"legs1_duration\").fill_null(0)).log1p().alias(\"log_leg1_duration\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n            .then((pl.col(\"legs0_duration\") - pl.col(\"legs1_duration\")).abs())\n            .otherwise(0).alias(\"duration_difference\"),\n        \n        # === ORIGINAL TRIP TYPE ===\n        (pl.col(\"legs1_duration\").is_null() | \n         (pl.col(\"legs1_duration\") == 0) | \n         pl.col(\"legs1_segments0_departureFrom_airport_iata\").is_null()).cast(pl.Int32).alias(\"is_one_way\"),\n        \n        # === ORIGINAL SEGMENT COUNT ===\n        (pl.sum_horizontal(pl.col(col).is_not_null().cast(pl.UInt8) for col in mc_exists) \n         if mc_exists else pl.lit(0)).alias(\"l0_seg\"),\n        \n        # === ORIGINAL FF FEATURES ===\n        (pl.col(\"frequentFlyer\").fill_null(\"\").str.count_matches(\"/\") + \n         (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\").cast(pl.Int32)).alias(\"n_ff_programs\"),\n        \n        # === ORIGINAL BINARY FEATURES ===\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        \n        # === ORIGINAL BAGGAGE & FEES (BASE) ===\n        (pl.col(\"legs0_segments0_baggageAllowance_quantity\").fill_null(0) + \n         pl.col(\"legs1_segments0_baggageAllowance_quantity\").fill_null(0)).alias(\"baggage_total\"),\n        (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + \n         pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n        (pl.col(\"miniRules0_monetaryAmount\").fill_null(0)).log1p().alias(\"log_fees0\"),\n        (pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).log1p().alias(\"log_fees1\"),\n        \n        # === ORIGINAL ROUTES & CARRIERS ===\n        pl.col(\"searchRoute\").is_in([\"MOWLED/LEDMOW\", \"LEDMOW/MOWLED\", \"MOWLED\", \"LEDMOW\", \"MOWAER/AERMOW\"])\n            .cast(pl.Int32).alias(\"is_popular_route\"),\n        \n        # === NEW ENHANCED ROUTE FEATURES ===\n        pl.col(\"searchRoute\").str.contains(\"MOW\").fill_null(False).cast(pl.Int32).alias(\"has_moscow\"),\n        pl.col(\"searchRoute\").str.contains(\"LED\").fill_null(False).cast(pl.Int32).alias(\"has_stpetersburg\"),\n        pl.col(\"searchRoute\").str.contains(\"AER\").fill_null(False).cast(pl.Int32).alias(\"has_sochi\"),\n        pl.col(\"searchRoute\").str.contains(\"SVO\").fill_null(False).cast(pl.Int32).alias(\"has_sheremetyevo\"),\n        pl.col(\"searchRoute\").str.contains(\"VKO\").fill_null(False).cast(pl.Int32).alias(\"has_vnukovo\"),\n        (pl.col(\"searchRoute\").str.count_matches(\"/\") + 1).alias(\"route_complexity\"),\n        pl.col(\"searchRoute\").str.len_chars().alias(\"route_length\"),\n        \n        # === ORIGINAL CABIN ===\n        pl.mean_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"avg_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) - \n         pl.col(\"legs1_segments0_cabinClass\").fill_null(0)).alias(\"cabin_class_diff\"),\n         \n        # === NEW ENHANCED CABIN FEATURES ===\n        pl.max_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"max_cabin_class\"),\n        pl.min_horizontal([\"legs0_segments0_cabinClass\", \"legs1_segments0_cabinClass\"]).alias(\"min_cabin_class\"),\n        (pl.col(\"legs0_segments0_cabinClass\").fill_null(0) >= 3).cast(pl.Int32).alias(\"has_business_leg0\"),\n        (pl.col(\"legs1_segments0_cabinClass\").fill_null(0) >= 3).cast(pl.Int32).alias(\"has_business_leg1\"),\n])\n\n# STEP 2: Add features that depend on columns created in step 1\ndf = df.with_columns([\n        # === ENHANCED BAGGAGE & FEES (using baggage_total and total_fees) ===\n        pl.when(pl.col(\"baggage_total\") > 0)\n            .then(pl.col(\"total_fees\") / pl.col(\"baggage_total\"))\n            .otherwise(0).alias(\"fee_per_baggage\"),\n])\n\nprint(\"Basic features created...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:14:10.052826Z","iopub.execute_input":"2025-08-16T05:14:10.053194Z","iopub.status.idle":"2025-08-16T05:14:12.001388Z","shell.execute_reply.started":"2025-08-16T05:14:10.053165Z","shell.execute_reply":"2025-08-16T05:14:11.996600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL SEGMENT COUNTS - Enhanced with additional analysis\nprint(\"Processing segment features...\")\n\n# First, ensure all duration columns are numeric\nduration_cols_to_check = []\nfor leg in (0, 1):\n    for seg in range(4):\n        col_name = f\"legs{leg}_segments{seg}_duration\"\n        if col_name in df.columns:\n            duration_cols_to_check.append(col_name)\n\n# Convert any string duration columns to numeric\nconversion_exprs = []\nfor col in duration_cols_to_check:\n    # Check if column exists and convert to numeric if it's string type\n    if col in df.columns:\n        col_dtype = df.select(pl.col(col)).dtypes[0]\n        if col_dtype in [pl.Utf8, pl.String]:\n            print(f\"Converting {col} from {col_dtype} to numeric\")\n            # Try to convert string to numeric, fallback to 0\n            conversion_exprs.append(\n                pl.col(col).cast(pl.Float64, strict=False).fill_null(0).alias(col)\n            )\n        elif col_dtype not in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]:\n            print(f\"Converting {col} from {col_dtype} to numeric\")\n            conversion_exprs.append(\n                pl.col(col).cast(pl.Float64, strict=False).fill_null(0).alias(col)\n            )\n\nif conversion_exprs:\n    df = df.with_columns(conversion_exprs)\n    print(f\"Converted {len(conversion_exprs)} duration columns to numeric\")\n\n# Now process segment features safely\nseg_exprs = []\nfor leg in (0, 1):\n    seg_cols = [f\"legs{leg}_segments{s}_duration\" for s in range(4) if f\"legs{leg}_segments{s}_duration\" in df.columns]\n    if seg_cols:\n        # Count non-null segments\n        seg_exprs.append(\n            pl.sum_horizontal(pl.col(c).is_not_null() for c in seg_cols)\n                .cast(pl.Int32).alias(f\"n_segments_leg{leg}\")\n        )\n        \n        # Average segment duration per leg - ensure numeric operations\n        try:\n            seg_exprs.append(\n                (pl.sum_horizontal(pl.col(c).fill_null(0).cast(pl.Float64) for c in seg_cols) / \n                 pl.sum_horizontal(pl.col(c).is_not_null().cast(pl.Int32) for c in seg_cols).clip(lower_bound=1).cast(pl.Float64)\n                ).alias(f\"avg_segment_duration_leg{leg}\")\n            )\n        except:\n            print(f\"Warning: Failed to create avg_segment_duration_leg{leg}, using fallback\")\n            seg_exprs.append(pl.lit(0.0).alias(f\"avg_segment_duration_leg{leg}\"))\n        \n        # Max segment duration per leg - ensure numeric operations\n        try:\n            seg_exprs.append(\n                pl.max_horizontal([pl.col(c).fill_null(0).cast(pl.Float64) for c in seg_cols]).alias(f\"max_segment_duration_leg{leg}\")\n            )\n        except:\n            print(f\"Warning: Failed to create max_segment_duration_leg{leg}, using fallback\")\n            seg_exprs.append(pl.lit(0.0).alias(f\"max_segment_duration_leg{leg}\"))\n    else:\n        seg_exprs.extend([\n            pl.lit(0).cast(pl.Int32).alias(f\"n_segments_leg{leg}\"),\n            pl.lit(0.0).alias(f\"avg_segment_duration_leg{leg}\"),\n            pl.lit(0.0).alias(f\"max_segment_duration_leg{leg}\")\n        ])\n\n# Add segment-based features with error handling\ntry:\n    df = df.with_columns(seg_exprs)\n    print(f\"Added {len(seg_exprs)} segment features successfully\")\nexcept Exception as e:\n    print(f\"Error adding segment features: {e}\")\n    # Add fallback features\n    fallback_exprs = [\n        pl.lit(0).cast(pl.Int32).alias(\"n_segments_leg0\"),\n        pl.lit(0).cast(pl.Int32).alias(\"n_segments_leg1\"),\n        pl.lit(0.0).alias(\"avg_segment_duration_leg0\"),\n        pl.lit(0.0).alias(\"avg_segment_duration_leg1\"),\n        pl.lit(0.0).alias(\"max_segment_duration_leg0\"),\n        pl.lit(0.0).alias(\"max_segment_duration_leg1\")\n    ]\n    df = df.with_columns(fallback_exprs)\n    print(\"Added fallback segment features\")\n\n# ORIGINAL DERIVED FEATURES - Enhanced\ndf = df.with_columns([\n    # === ORIGINAL ===\n    (pl.col(\"n_segments_leg0\") + pl.col(\"n_segments_leg1\")).alias(\"total_segments\"),\n    (pl.col(\"n_segments_leg0\") == 1).cast(pl.Int32).alias(\"is_direct_leg0\"),\n    pl.when(pl.col(\"is_one_way\") == 1).then(0)\n        .otherwise((pl.col(\"n_segments_leg1\") == 1).cast(pl.Int32)).alias(\"is_direct_leg1\"),\n        \n    # === NEW ENHANCED SEGMENT FEATURES ===\n    (pl.col(\"n_segments_leg0\") * pl.col(\"n_segments_leg1\")).alias(\"segment_interaction\"),\n    pl.when(pl.col(\"n_segments_leg1\") > 0)\n        .then(pl.col(\"n_segments_leg0\") / pl.col(\"n_segments_leg1\"))\n        .otherwise(1.0).alias(\"segment_ratio\"),\n    (pl.col(\"avg_segment_duration_leg0\") + pl.col(\"avg_segment_duration_leg1\") / 2).alias(\"overall_avg_segment_duration\"),\n])\n\n# ORIGINAL MORE DERIVED FEATURES - Enhanced\ndf = df.with_columns([\n    # === ORIGINAL ===\n    (pl.col(\"is_direct_leg0\") & pl.col(\"is_direct_leg1\")).cast(pl.Int32).alias(\"both_direct\"),\n    ((pl.col(\"isVip\") == 1) | (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_freq\"),\n    (pl.col(\"baggage_total\") > 0).cast(pl.Int32).alias(\"has_baggage\"),\n    (pl.col(\"total_fees\") > 0).cast(pl.Int32).alias(\"has_fees\"),\n    (pl.col(\"total_fees\") / (pl.col(\"totalPrice\") + 1)).alias(\"fee_rate\"),\n    pl.col(\"Id\").count().over(\"ranker_id\").alias(\"group_size\"),\n    \n    # === NEW ENHANCED FEATURES ===\n    (pl.col(\"baggage_total\") >= 20).cast(pl.Int32).alias(\"has_checked_baggage\"),\n    (pl.col(\"n_ff_programs\") >= 2).cast(pl.Int32).alias(\"multi_ff_programs\"),\n    ((pl.col(\"isVip\") == 1) & (pl.col(\"n_ff_programs\") > 0)).cast(pl.Int32).alias(\"is_vip_and_freq\"),\n    (pl.col(\"total_duration\") / (pl.col(\"total_segments\").clip(lower_bound=1))).alias(\"duration_per_segment\"),\n    (pl.col(\"totalPrice\") / (pl.col(\"total_duration\") + 1)).alias(\"price_per_minute\"),\n])\n\nprint(\"Enhanced segment and derived features created...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:14:15.238437Z","iopub.execute_input":"2025-08-16T05:14:15.238801Z","iopub.status.idle":"2025-08-16T05:14:16.840502Z","shell.execute_reply.started":"2025-08-16T05:14:15.238771Z","shell.execute_reply":"2025-08-16T05:14:16.835783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL MAJOR CARRIER - Enhanced with more carrier analysis\nif \"legs0_segments0_marketingCarrier_code\" in df.columns:\n    df = df.with_columns([\n        # Original\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\", \"S7\", \"U6\"])\n            .cast(pl.Int32).alias(\"is_major_carrier\"),\n        # Enhanced\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"SU\"]).cast(pl.Int32).alias(\"is_aeroflot\"),\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"S7\"]).cast(pl.Int32).alias(\"is_s7\"),\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"U6\"]).cast(pl.Int32).alias(\"is_ural\"),\n        # Low cost carriers\n        pl.col(\"legs0_segments0_marketingCarrier_code\").is_in([\"U6\", \"DP\", \"5N\"]).cast(pl.Int32).alias(\"is_low_cost\"),\n    ])\n    \n    # Cross-leg carrier consistency\n    if \"legs1_segments0_marketingCarrier_code\" in df.columns:\n        df = df.with_columns([\n            (pl.col(\"legs0_segments0_marketingCarrier_code\") == pl.col(\"legs1_segments0_marketingCarrier_code\"))\n                .fill_null(True).cast(pl.Int32).alias(\"same_carrier_both_legs\"),\n        ])\n    else:\n        df = df.with_columns(pl.lit(1).alias(\"same_carrier_both_legs\"))\nelse:\n    df = df.with_columns([\n        pl.lit(0).alias(\"is_major_carrier\"),\n        pl.lit(0).alias(\"is_aeroflot\"),\n        pl.lit(0).alias(\"is_s7\"),\n        pl.lit(0).alias(\"is_ural\"),\n        pl.lit(0).alias(\"is_low_cost\"),\n        pl.lit(1).alias(\"same_carrier_both_legs\"),\n    ])\n\n# Enhanced group size features\ndf = df.with_columns([\n    pl.col(\"group_size\").log1p().alias(\"group_size_log\"),\n    pl.col(\"group_size\").sqrt().alias(\"group_size_sqrt\"),\n    # Group size categories\n    pl.when(pl.col(\"group_size\") <= 5).then(0)\n        .when(pl.col(\"group_size\") <= 15).then(1)\n        .when(pl.col(\"group_size\") <= 30).then(2)\n        .otherwise(3).alias(\"group_size_category\")\n])\n\nprint(\"Carrier and group features enhanced...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:14:20.343132Z","iopub.execute_input":"2025-08-16T05:14:20.343472Z","iopub.status.idle":"2025-08-16T05:14:21.381757Z","shell.execute_reply.started":"2025-08-16T05:14:20.343445Z","shell.execute_reply":"2025-08-16T05:14:21.376664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_exprs = []\nfor col in (\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"):\n    if col in df.columns:\n        dt = pl.col(col).str.to_datetime(strict=False)\n        h = dt.dt.hour().fill_null(12)\n        \n        # Original features\n        time_exprs.extend([\n            h.alias(f\"{col}_hour\"),\n            dt.dt.weekday().fill_null(0).alias(f\"{col}_weekday\"),\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_business_time\")\n        ])\n        \n        # NEW: Enhanced time features\n        time_exprs.extend([\n            dt.dt.month().fill_null(6).alias(f\"{col}_month\"),\n            dt.dt.day().fill_null(15).alias(f\"{col}_day\"),\n            dt.dt.quarter().fill_null(2).alias(f\"{col}_quarter\"),\n            (dt.dt.weekday() >= 5).fill_null(False).cast(pl.Int32).alias(f\"{col}_is_weekend\"),\n            \n            # Time of day categories\n            pl.when(h < 6).then(0)  # Night\n                .when(h < 12).then(1)  # Morning  \n                .when(h < 18).then(2)  # Afternoon\n                .otherwise(3).alias(f\"{col}_time_period\"),\n            \n            # Peak travel times\n            (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{col}_peak_time\"),\n            ((h >= 22) | (h <= 5)).cast(pl.Int32).alias(f\"{col}_red_eye\"),\n            ((h >= 5) & (h <= 7)).cast(pl.Int32).alias(f\"{col}_early_morning\"),\n            \n            # Holiday periods (approximate Russian holidays)\n            ((dt.dt.month() == 12) & (dt.dt.day() >= 25)).fill_null(False).cast(pl.Int32).alias(f\"{col}_new_year_period\"),\n            ((dt.dt.month() == 1) & (dt.dt.day() <= 8)).fill_null(False).cast(pl.Int32).alias(f\"{col}_january_holidays\"),\n            ((dt.dt.month() == 5) & (dt.dt.day() <= 9)).fill_null(False).cast(pl.Int32).alias(f\"{col}_may_holidays\"),\n            \n            # Season indicators\n            pl.when(dt.dt.month().is_in([12, 1, 2])).then(0)  # Winter\n                .when(dt.dt.month().is_in([3, 4, 5])).then(1)  # Spring\n                .when(dt.dt.month().is_in([6, 7, 8])).then(2)  # Summer\n                .otherwise(3).alias(f\"{col}_season\"),\n        ])\n\nif time_exprs:\n    df = df.with_columns(time_exprs)\n\n# NEW: Cross-time features\nif all(col in df.columns for col in [\"legs0_departureAt\", \"legs0_arrivalAt\"]):\n    df = df.with_columns([\n        # Flight timing patterns\n        (pl.col(\"legs0_departureAt_hour\") == pl.col(\"legs0_arrivalAt_hour\")).cast(pl.Int32).alias(\"same_hour_leg0\"),\n        ((pl.col(\"legs0_departureAt_hour\") < 12) & (pl.col(\"legs0_arrivalAt_hour\") >= 12)).cast(pl.Int32).alias(\"morning_to_afternoon_leg0\"),\n        \n        # Travel day patterns\n        (pl.col(\"legs0_departureAt_weekday\") == pl.col(\"legs0_arrivalAt_weekday\")).cast(pl.Int32).alias(\"same_day_leg0\"),\n    ])\n    \n    # Connection time if both legs exist\n    if \"legs1_departureAt\" in df.columns:\n        df = df.with_columns([\n            ((pl.col(\"legs1_departureAt\").str.to_datetime(strict=False) - \n              pl.col(\"legs0_arrivalAt\").str.to_datetime(strict=False)).dt.total_minutes() / 60\n            ).fill_null(0).clip(lower_bound=0, upper_bound=48).alias(\"connection_time_hours\"),\n        ])\n        \n        # Connection time categories\n        df = df.with_columns([\n            (pl.col(\"connection_time_hours\") <= 2).cast(pl.Int32).alias(\"tight_connection\"),\n            ((pl.col(\"connection_time_hours\") > 2) & (pl.col(\"connection_time_hours\") <= 6)).cast(pl.Int32).alias(\"normal_connection\"),\n            (pl.col(\"connection_time_hours\") > 12).cast(pl.Int32).alias(\"overnight_layover\"),\n        ])\n    else:\n        df = df.with_columns([\n            pl.lit(0.0).alias(\"connection_time_hours\"),\n            pl.lit(0).alias(\"tight_connection\"),\n            pl.lit(0).alias(\"normal_connection\"),\n            pl.lit(0).alias(\"overnight_layover\"),\n        ])\n\nprint(\"Enhanced time features created...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:14:25.717252Z","iopub.execute_input":"2025-08-16T05:14:25.717575Z","iopub.status.idle":"2025-08-16T05:15:43.175254Z","shell.execute_reply.started":"2025-08-16T05:14:25.717548Z","shell.execute_reply":"2025-08-16T05:15:43.170816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NEW: Advanced Aircraft Features\nif \"legs0_segments0_aircraft_code\" in df.columns:\n    # Wide body aircraft\n    widebody_codes = ['77W', '777', '330', '340', '350', '380', '787', 'A33', 'A34', 'A35', '763', '764', '767']\n    # Modern efficient aircraft\n    modern_codes = ['787', '350', 'A35', '38M', '32N', '321', '32Q']\n    \n    df = df.with_columns([\n        pl.col(\"legs0_segments0_aircraft_code\").is_in(widebody_codes).cast(pl.Int32).alias(\"is_widebody_leg0\"),\n        pl.col(\"legs0_segments0_aircraft_code\").is_in(modern_codes).cast(pl.Int32).alias(\"is_modern_aircraft_leg0\"),\n    ])\n    \n    if \"legs1_segments0_aircraft_code\" in df.columns:\n        df = df.with_columns([\n            pl.col(\"legs1_segments0_aircraft_code\").is_in(widebody_codes).cast(pl.Int32).alias(\"is_widebody_leg1\"),\n            pl.col(\"legs1_segments0_aircraft_code\").is_in(modern_codes).cast(pl.Int32).alias(\"is_modern_aircraft_leg1\"),\n            (pl.col(\"legs0_segments0_aircraft_code\") == pl.col(\"legs1_segments0_aircraft_code\")).fill_null(False).cast(pl.Int32).alias(\"same_aircraft_type\")\n        ])\n    else:\n        df = df.with_columns([\n            pl.lit(0).alias(\"is_widebody_leg1\"),\n            pl.lit(0).alias(\"is_modern_aircraft_leg1\"),\n            pl.lit(0).alias(\"same_aircraft_type\")\n        ])\nelse:\n    df = df.with_columns([\n        pl.lit(0).alias(\"is_widebody_leg0\"),\n        pl.lit(0).alias(\"is_modern_aircraft_leg0\"),\n        pl.lit(0).alias(\"is_widebody_leg1\"),\n        pl.lit(0).alias(\"is_modern_aircraft_leg1\"),\n        pl.lit(0).alias(\"same_aircraft_type\")\n    ])\n\nprint(\"Aircraft features enhanced...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:15:46.960894Z","iopub.execute_input":"2025-08-16T05:15:46.961279Z","iopub.status.idle":"2025-08-16T05:15:47.487032Z","shell.execute_reply.started":"2025-08-16T05:15:46.961246Z","shell.execute_reply":"2025-08-16T05:15:47.482443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 1: ORIGINAL BATCH RANK COMPUTATIONS - Enhanced\nprint(\"Creating ranking features...\")\n\nrank_exprs = []\nfor col, alias in [(\"totalPrice\", \"price\"), (\"total_duration\", \"duration\")]:\n    # Original ranks\n    rank_exprs.append(pl.col(col).rank().over(\"ranker_id\").alias(f\"{alias}_rank\"))\n    \n    # NEW: Enhanced ranking features\n    rank_exprs.extend([\n        pl.col(col).rank(method=\"average\").over(\"ranker_id\").alias(f\"{alias}_rank_avg\"),\n        pl.col(col).rank(method=\"dense\").over(\"ranker_id\").alias(f\"{alias}_rank_dense\"),\n        (pl.col(col).rank().over(\"ranker_id\") / pl.col(col).count().over(\"ranker_id\")).alias(f\"{alias}_rank_pct\"),\n    ])\n\n# STEP 1: Apply basic ranks first\ntry:\n    df = df.with_columns(rank_exprs)\n    print(f\" Added {len(rank_exprs)} basic ranking features\")\nexcept Exception as e:\n    print(f\" Error creating basic ranks: {e}\")\n    # Add fallback ranks\n    fallback_ranks = [\n        pl.lit(1).alias(\"price_rank\"),\n        pl.lit(1).alias(\"duration_rank\"),\n        pl.lit(1).alias(\"price_rank_avg\"),\n        pl.lit(1).alias(\"duration_rank_avg\"),\n        pl.lit(1).alias(\"price_rank_dense\"),\n        pl.lit(1).alias(\"duration_rank_dense\"),\n        pl.lit(0.5).alias(\"price_rank_pct\"),\n        pl.lit(0.5).alias(\"duration_rank_pct\"),\n    ]\n    df = df.with_columns(fallback_ranks)\n    print(\"Added fallback ranking features\")\n\n# STEP 2: PRICE-SPECIFIC FEATURES that depend on ranks - Enhanced\nprice_exprs = [\n    # Original\n    (pl.col(\"totalPrice\").rank(\"average\").over(\"ranker_id\") / \n     pl.col(\"totalPrice\").count().over(\"ranker_id\")).alias(\"price_pct_rank\"),\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_cheapest\"),\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").median().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_from_median\"),\n    (pl.col(\"l0_seg\") == pl.col(\"l0_seg\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_min_segments\"),\n    \n    # NEW: Enhanced price analysis\n    (pl.col(\"totalPrice\") == pl.col(\"totalPrice\").max().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_most_expensive\"),\n    (pl.col(\"totalPrice\") / pl.col(\"totalPrice\").min().over(\"ranker_id\")).alias(\"price_ratio_to_min\"),\n    (pl.col(\"totalPrice\") / pl.col(\"totalPrice\").max().over(\"ranker_id\")).alias(\"price_ratio_to_max\"),\n    (pl.col(\"totalPrice\") / pl.col(\"totalPrice\").mean().over(\"ranker_id\")).alias(\"price_ratio_to_mean\"),\n    \n    # Price z-score (standardized)\n    ((pl.col(\"totalPrice\") - pl.col(\"totalPrice\").mean().over(\"ranker_id\")) / \n     (pl.col(\"totalPrice\").std().over(\"ranker_id\") + 1)).alias(\"price_z_score\"),\n     \n    # Market spread analysis\n    (pl.col(\"totalPrice\").max().over(\"ranker_id\") - pl.col(\"totalPrice\").min().over(\"ranker_id\")).alias(\"price_range\"),\n    pl.col(\"totalPrice\").std().over(\"ranker_id\").alias(\"price_std\"),\n    \n    # Duration rankings\n    (pl.col(\"total_duration\") == pl.col(\"total_duration\").min().over(\"ranker_id\")).cast(pl.Int32).alias(\"is_fastest\"),\n    (pl.col(\"total_duration\") / pl.col(\"total_duration\").min().over(\"ranker_id\")).alias(\"duration_ratio_to_min\"),\n]\n\n# STEP 2: Apply price-specific features\ntry:\n    df = df.with_columns(price_exprs)\n    print(f\"Added {len(price_exprs)} price analysis features\")\nexcept Exception as e:\n    print(f\" Error creating price features: {e}\")\n    print(\"Continuing without advanced price features...\")\n\n# STEP 3: Combined value scores that depend on both price_rank and duration_rank\ncombined_exprs = [\n    # Combined value scores (now that we have price_rank and duration_rank)\n    ((1.0 / pl.col(\"price_rank\").clip(lower_bound=1)) + (1.0 / pl.col(\"duration_rank\").clip(lower_bound=1))).alias(\"combined_value_score\"),\n    (pl.col(\"price_rank\") * pl.col(\"duration_rank\")).alias(\"rank_product\"),\n]\n\n# STEP 3: Apply combined features\ntry:\n    df = df.with_columns(combined_exprs)\n    print(f\" Added {len(combined_exprs)} combined ranking features\")\nexcept Exception as e:\n    print(f\" Error creating combined features: {e}\")\n    # Add fallback combined features\n    fallback_combined = [\n        pl.lit(2.0).alias(\"combined_value_score\"),\n        pl.lit(1.0).alias(\"rank_product\"),\n    ]\n    df = df.with_columns(fallback_combined)\n    print(\"Added fallback combined features\")\n\nprint(\"Enhanced ranking features completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:15:50.074859Z","iopub.execute_input":"2025-08-16T05:15:50.075201Z","iopub.status.idle":"2025-08-16T05:15:57.208464Z","shell.execute_reply.started":"2025-08-16T05:15:50.075173Z","shell.execute_reply":"2025-08-16T05:15:57.203886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ENHANCED CHEAPEST DIRECT - Enhanced with more complex logic and error handling\nprint(\"Creating direct flight features...\")\n\ntry:\n    # Check if is_direct_leg0 exists\n    if \"is_direct_leg0\" not in df.columns:\n        print(\"Warning: is_direct_leg0 column not found, creating fallback\")\n        df = df.with_columns(pl.lit(0).cast(pl.Int32).alias(\"is_direct_leg0\"))\n    \n    # Calculate direct flight statistics\n    direct_cheapest = (\n        df.filter(pl.col(\"is_direct_leg0\") == 1)\n        .group_by(\"ranker_id\")\n        .agg([\n            pl.col(\"totalPrice\").min().alias(\"min_direct_price\"),\n            pl.col(\"total_duration\").min().alias(\"min_direct_duration\"),\n            pl.len().alias(\"n_direct_options\")\n        ])\n    )\n    \n    # STEP 1: Join the direct flight statistics\n    df = df.join(direct_cheapest, on=\"ranker_id\", how=\"left\")\n    print(\" Joined direct flight statistics\")\n    \n    # STEP 2: Create basic direct flight features (no self-references)\n    direct_features_1 = [\n        # Original\n        ((pl.col(\"is_direct_leg0\") == 1) & \n         (pl.col(\"totalPrice\") == pl.col(\"min_direct_price\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_cheapest\"),\n        \n        # NEW: Enhanced direct flight analysis\n        ((pl.col(\"is_direct_leg0\") == 1) & \n         (pl.col(\"total_duration\") == pl.col(\"min_direct_duration\"))).cast(pl.Int32).fill_null(0).alias(\"is_direct_fastest\"),\n        pl.col(\"n_direct_options\").fill_null(0).alias(\"direct_options_available\"),\n        pl.when(pl.col(\"min_direct_price\").is_not_null())\n            .then(pl.col(\"totalPrice\") / pl.col(\"min_direct_price\"))\n            .otherwise(1.0).alias(\"price_ratio_to_cheapest_direct\"),\n    ]\n    \n    df = df.with_columns(direct_features_1)\n    print(f\" Added {len(direct_features_1)} basic direct flight features\")\n    \n    # STEP 3: Create features that depend on columns from step 2\n    direct_features_2 = [\n        (pl.col(\"direct_options_available\") > 0).cast(pl.Int32).alias(\"has_direct_options\"),\n    ]\n    \n    df = df.with_columns(direct_features_2)\n    print(f\" Added {len(direct_features_2)} derived direct flight features\")\n    \n    # STEP 4: Clean up intermediate columns\n    df = df.drop([\"min_direct_price\", \"min_direct_duration\"])\n    print(\" Cleaned up intermediate columns\")\n\nexcept Exception as e:\n    print(f\" Error creating direct flight features: {e}\")\n    print(\"Adding fallback direct flight features...\")\n    \n    # Add fallback features\n    fallback_direct = [\n        pl.lit(0).cast(pl.Int32).alias(\"is_direct_cheapest\"),\n        pl.lit(0).cast(pl.Int32).alias(\"is_direct_fastest\"),\n        pl.lit(0).alias(\"direct_options_available\"),\n        pl.lit(0).cast(pl.Int32).alias(\"has_direct_options\"),\n        pl.lit(1.0).alias(\"price_ratio_to_cheapest_direct\"),\n    ]\n    df = df.with_columns(fallback_direct)\n    print(\"Added fallback direct flight features\")\n\nprint(\"Enhanced direct flight features completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:17:19.078230Z","iopub.execute_input":"2025-08-16T05:17:19.078559Z","iopub.status.idle":"2025-08-16T05:17:19.520978Z","shell.execute_reply.started":"2025-08-16T05:17:19.078531Z","shell.execute_reply":"2025-08-16T05:17:19.516389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL POPULARITY FEATURES - Enhanced with more sophisticated analysis\ndf = (\n    df.join(\n        train.group_by('legs0_segments0_marketingCarrier_code').agg([\n            pl.mean('selected').alias('carrier0_pop'),\n            pl.count().alias('carrier0_frequency'),\n            pl.std('selected').alias('carrier0_selection_variance')\n        ]),\n        on='legs0_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .join(\n        train.group_by('legs1_segments0_marketingCarrier_code').agg([\n            pl.mean('selected').alias('carrier1_pop'),\n            pl.count().alias('carrier1_frequency'),\n            pl.std('selected').alias('carrier1_selection_variance')\n        ]),\n        on='legs1_segments0_marketingCarrier_code', \n        how='left'\n    )\n    .with_columns([\n        # Original\n        pl.col('carrier0_pop').fill_null(0.0),\n        pl.col('carrier1_pop').fill_null(0.0),\n        \n        # NEW: Enhanced carrier popularity\n        pl.col('carrier0_frequency').fill_null(0),\n        pl.col('carrier1_frequency').fill_null(0),\n        pl.col('carrier0_selection_variance').fill_null(0.0),\n        pl.col('carrier1_selection_variance').fill_null(0.0),\n    ])\n)\n\n# ORIGINAL FINAL FEATURES - Enhanced\ndf = df.with_columns([\n    # Original\n    (pl.col('carrier0_pop') * pl.col('carrier1_pop')).alias('carrier_pop_product'),\n    \n    # NEW: Enhanced carrier analysis\n    (pl.col('carrier0_pop') + pl.col('carrier1_pop')).alias('carrier_pop_sum'),\n    ((pl.col('carrier0_pop') - pl.col('carrier1_pop')).abs()).alias('carrier_pop_diff'),\n    pl.max_horizontal(['carrier0_pop', 'carrier1_pop']).alias('max_carrier_pop'),\n    pl.min_horizontal(['carrier0_pop', 'carrier1_pop']).alias('min_carrier_pop'),\n    \n    # Market presence\n    (pl.col('carrier0_frequency') + pl.col('carrier1_frequency')).alias('total_carrier_frequency'),\n    (pl.col('carrier0_frequency') >= 1000).cast(pl.Int32).alias('carrier0_high_frequency'),\n    (pl.col('carrier1_frequency') >= 1000).cast(pl.Int32).alias('carrier1_high_frequency'),\n])\n\nprint(\"Enhanced popularity features created...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:17:22.412096Z","iopub.execute_input":"2025-08-16T05:17:22.412459Z","iopub.status.idle":"2025-08-16T05:17:27.956290Z","shell.execute_reply.started":"2025-08-16T05:17:22.412428Z","shell.execute_reply":"2025-08-16T05:17:27.951051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL POPULARITY FEATURE BASED ON ROUND TRIP - Enhanced with route analysis\nrequired_cols = [\n    \"legs0_segments0_departureFrom_airport_iata\",\n    \"legs0_segments0_arrivalTo_airport_iata\",\n    \"legs1_segments0_departureFrom_airport_iata\",\n    \"legs1_segments0_arrivalTo_airport_iata\"\n]\n\nif all(col in df.columns for col in required_cols):\n    # Original round trip route\n    df = df.with_columns([\n        (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n         pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\"),\n         \n        # NEW: Enhanced route analysis\n        (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs0_segments0_arrivalTo_airport_iata\")).alias(\"leg0_route\"),\n        (pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n         pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"leg1_route\"),\n         \n        # Route symmetry (true round trip vs open jaw)\n        ((pl.col(\"legs0_segments0_departureFrom_airport_iata\") == pl.col(\"legs1_segments0_arrivalTo_airport_iata\")) &\n         (pl.col(\"legs0_segments0_arrivalTo_airport_iata\") == pl.col(\"legs1_segments0_departureFrom_airport_iata\"))\n        ).cast(pl.Int32).alias(\"is_true_round_trip\"),\n    ])\n\n    # Calculate original round trip frequency\n    round_trip_freq = (\n        train.with_columns([\n            (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs0_segments0_arrivalTo_airport_iata\") + \"__\" +\n             pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"round_trip_route\")\n        ])\n        .group_by(\"round_trip_route\")\n        .agg([\n            pl.count().alias(\"rt_route_count\"),\n            pl.mean('selected').alias('rt_route_popularity')\n        ])\n    )\n    \n    # NEW: Individual leg route frequencies\n    leg0_freq = (\n        train.with_columns([\n            (pl.col(\"legs0_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs0_segments0_arrivalTo_airport_iata\")).alias(\"leg0_route\")\n        ])\n        .group_by(\"leg0_route\")\n        .agg([\n            pl.count().alias(\"leg0_route_count\"),\n            pl.mean('selected').alias('leg0_route_popularity')\n        ])\n    )\n    \n    leg1_freq = (\n        train.with_columns([\n            (pl.col(\"legs1_segments0_departureFrom_airport_iata\") + \"_\" + \n             pl.col(\"legs1_segments0_arrivalTo_airport_iata\")).alias(\"leg1_route\")\n        ])\n        .group_by(\"leg1_route\")\n        .agg([\n            pl.count().alias(\"leg1_route_count\"),\n            pl.mean('selected').alias('leg1_route_popularity')\n        ])\n    )\n\n    # Join all route features\n    df = (df\n        .join(round_trip_freq, on=\"round_trip_route\", how=\"left\")\n        .join(leg0_freq, on=\"leg0_route\", how=\"left\")\n        .join(leg1_freq, on=\"leg1_route\", how=\"left\")\n        .with_columns([\n            # Original\n            pl.col(\"rt_route_count\").fill_null(0).alias(\"round_trip_freq\"),\n            \n            # NEW: Enhanced route features\n            pl.col(\"rt_route_popularity\").fill_null(0.0).alias(\"round_trip_popularity\"),\n            pl.col(\"leg0_route_count\").fill_null(0).alias(\"leg0_route_freq\"),\n            pl.col(\"leg1_route_count\").fill_null(0).alias(\"leg1_route_freq\"),\n            pl.col(\"leg0_route_popularity\").fill_null(0.0).alias(\"leg0_route_popularity\"),\n            pl.col(\"leg1_route_popularity\").fill_null(0.0).alias(\"leg1_route_popularity\"),\n            \n            # Route frequency categories\n            (pl.col(\"rt_route_count\").fill_null(0) >= 100).cast(pl.Int32).alias(\"is_popular_round_trip\"),\n            (pl.col(\"leg0_route_count\").fill_null(0) >= 500).cast(pl.Int32).alias(\"is_popular_leg0_route\"),\n            (pl.col(\"leg1_route_count\").fill_null(0) >= 500).cast(pl.Int32).alias(\"is_popular_leg1_route\"),\n        ])\n        .drop([\"round_trip_route\", \"leg0_route\", \"leg1_route\", \"rt_route_count\", \"rt_route_popularity\",\n               \"leg0_route_count\", \"leg1_route_count\", \"leg0_route_popularity\", \"leg1_route_popularity\"])\n    )\nelse:\n    df = df.with_columns([\n        pl.lit(0).alias(\"round_trip_freq\"),\n        pl.lit(0.0).alias(\"round_trip_popularity\"),\n        pl.lit(0).alias(\"leg0_route_freq\"),\n        pl.lit(0).alias(\"leg1_route_freq\"),\n        pl.lit(0.0).alias(\"leg0_route_popularity\"),\n        pl.lit(0.0).alias(\"leg1_route_popularity\"),\n        pl.lit(0).alias(\"is_true_round_trip\"),\n        pl.lit(0).alias(\"is_popular_round_trip\"),\n        pl.lit(0).alias(\"is_popular_leg0_route\"),\n        pl.lit(0).alias(\"is_popular_leg1_route\"),\n    ])\n\nprint(\"Enhanced route analysis completed...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:17:31.074863Z","iopub.execute_input":"2025-08-16T05:17:31.075251Z","iopub.status.idle":"2025-08-16T05:17:38.334940Z","shell.execute_reply.started":"2025-08-16T05:17:31.075220Z","shell.execute_reply":"2025-08-16T05:17:38.330004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NEW: Advanced Interaction Features\ninteraction_features = [\n    # Price-time interactions\n    (pl.col(\"totalPrice\") * pl.col(\"legs0_departureAt_peak_time\")).alias(\"price_peak_time_interaction\"),\n    (pl.col(\"totalPrice\") * pl.col(\"legs0_departureAt_is_weekend\")).alias(\"price_weekend_interaction\"),\n    (pl.col(\"totalPrice\") * pl.col(\"is_direct_leg0\")).alias(\"price_direct_interaction\"),\n    \n    # Duration-segment interactions  \n    (pl.col(\"total_duration\") * pl.col(\"total_segments\")).alias(\"duration_segments_interaction\"),\n    (pl.col(\"total_duration\") * pl.col(\"is_major_carrier\")).alias(\"duration_major_carrier_interaction\"),\n    \n    # Service level interactions\n    (pl.col(\"avg_cabin_class\") * pl.col(\"totalPrice\")).alias(\"cabin_price_interaction\"),\n    (pl.col(\"baggage_total\") * pl.col(\"total_fees\")).alias(\"baggage_fees_interaction\"),\n    (pl.col(\"is_vip_freq\") * pl.col(\"avg_cabin_class\")).alias(\"vip_cabin_interaction\"),\n    \n    # Market competition interactions\n    (pl.col(\"group_size\") * pl.col(\"price_std\")).alias(\"competition_variance_interaction\"),\n    (pl.col(\"carrier_pop_product\") * pl.col(\"totalPrice\")).alias(\"popularity_price_interaction\"),\n    \n    # Route-carrier interactions\n    (pl.col(\"is_popular_route\") * pl.col(\"is_major_carrier\")).alias(\"popular_route_major_carrier\"),\n    (pl.col(\"round_trip_freq\") * pl.col(\"carrier0_pop\")).alias(\"route_freq_popularity_interaction\"),\n]\n\ndf = df.with_columns(interaction_features)\n\nprint(\"Advanced interaction features created...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:17:42.037255Z","iopub.execute_input":"2025-08-16T05:17:42.037560Z","iopub.status.idle":"2025-08-16T05:17:42.225421Z","shell.execute_reply.started":"2025-08-16T05:17:42.037534Z","shell.execute_reply":"2025-08-16T05:17:42.221472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NEW: Booking Pattern Features with type checking\nprint(\"Creating booking pattern features...\")\n\nif 'requestDate' in df.columns:\n    try:\n        # Check the data type of requestDate column\n        request_date_dtype = df.select(pl.col('requestDate')).dtypes[0]\n        print(f\"requestDate column type: {request_date_dtype}\")\n        \n        # Convert to datetime based on current type\n        if request_date_dtype in [pl.Utf8, pl.String]:\n            # String type - convert from string\n            df = df.with_columns([\n                pl.col('requestDate').str.to_datetime(strict=False).alias('request_dt')\n            ])\n        elif request_date_dtype in [pl.Datetime, pl.Date]:\n            # Already datetime type - just alias it\n            df = df.with_columns([\n                pl.col('requestDate').alias('request_dt')\n            ])\n        else:\n            # Unknown type - try to cast\n            print(f\"Warning: Unexpected requestDate type {request_date_dtype}, attempting conversion\")\n            df = df.with_columns([\n                pl.col('requestDate').cast(pl.Datetime, strict=False).alias('request_dt')\n            ])\n        \n        print(\" Successfully converted requestDate to datetime\")\n        \n        # Request timing features\n        df = df.with_columns([\n            pl.col('request_dt').dt.hour().alias('request_hour'),\n            pl.col('request_dt').dt.weekday().alias('request_weekday'),\n            pl.col('request_dt').dt.month().alias('request_month'),\n            (pl.col('request_dt').dt.weekday() >= 5).cast(pl.Int32).alias('request_weekend'),\n            ((pl.col('request_dt').dt.hour() >= 9) & (pl.col('request_dt').dt.hour() <= 17)).cast(pl.Int32).alias('request_business_hours'),\n        ])\n        \n        print(\" Added request timing features\")\n        \n        # Lead time calculation\n        if 'legs0_departureAt' in df.columns:\n            try:\n                # Check legs0_departureAt type\n                departure_dtype = df.select(pl.col('legs0_departureAt')).dtypes[0]\n                print(f\"legs0_departureAt column type: {departure_dtype}\")\n                \n                if departure_dtype in [pl.Utf8, pl.String]:\n                    # String type - convert from string\n                    lead_time_expr = (\n                        pl.col('legs0_departureAt').str.to_datetime(strict=False) - pl.col('request_dt')\n                    ).dt.total_days().clip(lower_bound=0, upper_bound=365).alias('booking_lead_days')\n                else:\n                    # Already datetime type\n                    lead_time_expr = (\n                        pl.col('legs0_departureAt') - pl.col('request_dt')\n                    ).dt.total_days().clip(lower_bound=0, upper_bound=365).alias('booking_lead_days')\n                \n                df = df.with_columns([lead_time_expr])\n                print(\" Calculated booking lead days\")\n                \n                # Lead time categories\n                df = df.with_columns([\n                    (pl.col('booking_lead_days') <= 1).cast(pl.Int32).alias('same_day_booking'),\n                    (pl.col('booking_lead_days') <= 7).cast(pl.Int32).alias('week_ahead_booking'),\n                    (pl.col('booking_lead_days') >= 30).cast(pl.Int32).alias('advance_booking'),\n                    (pl.col('booking_lead_days') >= 60).cast(pl.Int32).alias('far_advance_booking'),\n                ])\n                print(\" Added lead time categories\")\n                \n            except Exception as e:\n                print(f\" Error calculating lead time: {e}\")\n                # Fallback lead time features\n                df = df.with_columns([\n                    pl.lit(7.0).alias('booking_lead_days'),\n                    pl.lit(0).alias('same_day_booking'),\n                    pl.lit(1).alias('week_ahead_booking'),\n                    pl.lit(0).alias('advance_booking'),\n                    pl.lit(0).alias('far_advance_booking'),\n                ])\n                print(\"Added fallback lead time features\")\n        else:\n            df = df.with_columns([\n                pl.lit(7.0).alias('booking_lead_days'),\n                pl.lit(0).alias('same_day_booking'),\n                pl.lit(1).alias('week_ahead_booking'),\n                pl.lit(0).alias('advance_booking'),\n                pl.lit(0).alias('far_advance_booking'),\n            ])\n            print(\"Added default lead time features (legs0_departureAt not found)\")\n        \n        # Clean up intermediate column\n        df = df.drop('request_dt')\n        print(\" Cleaned up intermediate datetime column\")\n\n    except Exception as e:\n        print(f\" Error processing requestDate: {e}\")\n        print(\"Adding fallback booking features...\")\n        # Add fallback features\n        df = df.with_columns([\n            pl.lit(12).alias('request_hour'),\n            pl.lit(2).alias('request_weekday'),\n            pl.lit(6).alias('request_month'),\n            pl.lit(0).alias('request_weekend'),\n            pl.lit(1).alias('request_business_hours'),\n            pl.lit(7.0).alias('booking_lead_days'),\n            pl.lit(0).alias('same_day_booking'),\n            pl.lit(1).alias('week_ahead_booking'),\n            pl.lit(0).alias('advance_booking'),\n            pl.lit(0).alias('far_advance_booking'),\n        ])\n        print(\"Added fallback booking features\")\n        \nelse:\n    # Default values if requestDate not available\n    print(\"requestDate column not found, using default values\")\n    df = df.with_columns([\n        pl.lit(12).alias('request_hour'),\n        pl.lit(2).alias('request_weekday'),\n        pl.lit(6).alias('request_month'),\n        pl.lit(0).alias('request_weekend'),\n        pl.lit(1).alias('request_business_hours'),\n        pl.lit(7.0).alias('booking_lead_days'),\n        pl.lit(0).alias('same_day_booking'),\n        pl.lit(1).alias('week_ahead_booking'),\n        pl.lit(0).alias('advance_booking'),\n        pl.lit(0).alias('far_advance_booking'),\n    ])\n\nprint(\"Booking pattern features completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:17:46.912429Z","iopub.execute_input":"2025-08-16T05:17:46.912725Z","iopub.status.idle":"2025-08-16T05:17:57.486399Z","shell.execute_reply.started":"2025-08-16T05:17:46.912693Z","shell.execute_reply":"2025-08-16T05:17:57.479584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL FILL NULLS - Enhanced with smarter filling strategies\n# Get all numeric columns\nnumeric_cols = df.select(pl.selectors.numeric()).columns\nstring_cols = df.select(pl.selectors.string()).columns\n\n# Smart null filling for numeric columns\nnumeric_fill_exprs = []\nfor col in numeric_cols:\n    null_count = df[col].is_null().sum()\n    if null_count > 0:\n        if col.endswith('_pop') or col.endswith('_popularity'):\n            # Popularity metrics: fill with median\n            fill_value = df[col].median()\n        elif 'price' in col.lower() or 'fee' in col.lower() or 'cost' in col.lower():\n            # Price-related: fill with 0\n            fill_value = 0\n        elif 'duration' in col.lower() or 'time' in col.lower():\n            # Duration-related: fill with 0\n            fill_value = 0\n        elif col.endswith('_std') or col.endswith('_variance'):\n            # Variance metrics: fill with 0\n            fill_value = 0\n        elif col.endswith('_count') or col.endswith('_frequency') or col.endswith('_freq'):\n            # Count metrics: fill with 0\n            fill_value = 0\n        else:\n            # Default: fill with 0\n            fill_value = 0\n        \n        numeric_fill_exprs.append(pl.col(col).fill_null(fill_value))\n\n# Fill string columns\nstring_fill_exprs = [pl.col(c).fill_null(\"missing\") for c in string_cols]\n\n# Apply all fills\ndata = df.with_columns(numeric_fill_exprs + string_fill_exprs)\n\nprint(f\"Filled nulls in {len(numeric_fill_exprs)} numeric and {len(string_fill_exprs)} string columns\")\nprint(f\"Final data shape: {data.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:18:00.918309Z","iopub.execute_input":"2025-08-16T05:18:00.918665Z","iopub.status.idle":"2025-08-16T05:18:02.971375Z","shell.execute_reply.started":"2025-08-16T05:18:00.918637Z","shell.execute_reply":"2025-08-16T05:18:02.967812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Enhanced Feature Selection","metadata":{}},{"cell_type":"code","source":"# ORIGINAL CATEGORICAL FEATURES - Enhanced with new categoricals\ncat_features = [\n    # Original core categoricals\n    'nationality', 'searchRoute', 'corporateTariffCode',\n    'bySelf', 'sex', 'companyID',\n    \n    # Leg 0 segments 0-1\n    'legs0_segments0_aircraft_code', 'legs0_segments0_arrivalTo_airport_city_iata',\n    'legs0_segments0_arrivalTo_airport_iata', 'legs0_segments0_departureFrom_airport_iata',\n    'legs0_segments0_marketingCarrier_code', 'legs0_segments0_operatingCarrier_code',\n    'legs0_segments0_flightNumber',\n    'legs0_segments1_aircraft_code', 'legs0_segments1_arrivalTo_airport_city_iata',\n    'legs0_segments1_arrivalTo_airport_iata', 'legs0_segments1_departureFrom_airport_iata',\n    'legs0_segments1_marketingCarrier_code', 'legs0_segments1_operatingCarrier_code',\n    'legs0_segments1_flightNumber',\n    \n    # Leg 1 segments 0-1\n    'legs1_segments0_aircraft_code', 'legs1_segments0_arrivalTo_airport_city_iata',\n    'legs1_segments0_arrivalTo_airport_iata', 'legs1_segments0_departureFrom_airport_iata',\n    'legs1_segments0_marketingCarrier_code', 'legs1_segments0_operatingCarrier_code',\n    'legs1_segments0_flightNumber',\n    'legs1_segments1_aircraft_code', 'legs1_segments1_arrivalTo_airport_city_iata',\n    'legs1_segments1_arrivalTo_airport_iata', 'legs1_segments1_departureFrom_airport_iata',\n    'legs1_segments1_marketingCarrier_code', 'legs1_segments1_operatingCarrier_code',\n    'legs1_segments1_flightNumber',\n    \n    # NEW: Enhanced categoricals\n    'frequentFlyer',  # Re-include for encoding\n    'group_size_category',  # New categorical\n    \n    # Time-based categoricals\n    'legs0_departureAt_time_period', 'legs0_arrivalAt_time_period',\n    'legs1_departureAt_time_period', 'legs1_arrivalAt_time_period',\n    'legs0_departureAt_season', 'legs1_departureAt_season',\n    'request_month', 'request_weekday',\n]\n\n# ORIGINAL COLUMNS TO EXCLUDE - Enhanced exclusion list\nexclude_cols = [\n    # Original exclusions\n    'Id', 'ranker_id', 'selected', 'profileId', 'requestDate',\n    'legs0_departureAt', 'legs0_arrivalAt', 'legs1_departureAt', 'legs1_arrivalAt',\n    'pricingInfo_passengerCount',  # Constant\n    \n    # NEW: Additional exclusions\n    # Remove intermediate computation columns that shouldn't be features\n    'carrier0_frequency', 'carrier1_frequency', \n    'carrier0_selection_variance', 'carrier1_selection_variance',\n    'n_direct_options',  # Leakage risk\n    \n    # Remove redundant ranking columns (keep only the most important ones)\n    'price_rank_dense', 'duration_rank_dense',\n    'price_rank_avg', 'duration_rank_avg',\n]\n\n# ORIGINAL SEGMENT EXCLUSIONS - Enhanced\nfor leg in [0, 1]:\n    for seg in [2, 3]:\n        for suffix in ['aircraft_code', 'arrivalTo_airport_city_iata', 'arrivalTo_airport_iata',\n                      'baggageAllowance_quantity', 'baggageAllowance_weightMeasurementType',\n                      'cabinClass', 'departureFrom_airport_iata', 'duration', 'flightNumber',\n                      'marketingCarrier_code', 'operatingCarrier_code', 'seatsAvailable']:\n            exclude_cols.append(f'legs{leg}_segments{seg}_{suffix}')\n\n# Final feature selection\nfeature_cols = [col for col in data.columns if col not in exclude_cols]\ncat_features_final = [col for col in cat_features if col in feature_cols]\n\n# Remove features with too many nulls or too little variance\nlow_variance_features = []\nfor col in feature_cols:\n    if col in data.columns:\n        if data[col].dtype in [pl.Float64, pl.Float32, pl.Int64, pl.Int32]:\n            # Check variance\n            var = data[col].var()\n            if var is not None and var < 1e-10:\n                low_variance_features.append(col)\n        else:\n            # Check unique values for categorical\n            unique_count = data[col].n_unique()\n            if unique_count <= 1:\n                low_variance_features.append(col)\n\n# Remove low variance features\nfeature_cols = [col for col in feature_cols if col not in low_variance_features]\ncat_features_final = [col for col in cat_features_final if col not in low_variance_features]\n\nprint(f\"Removed {len(low_variance_features)} low variance features: {low_variance_features[:10]}...\")\nprint(f\"Using {len(feature_cols)} features ({len(cat_features_final)} categorical)\")\n\nX = data.select(feature_cols)\ny = data.select('selected')\ngroups = data.select('ranker_id')\n\nprint(f\"Final feature matrix shape: {X.shape}\")\nprint(f\"Number of groups: {groups.n_unique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:18:15.642199Z","iopub.execute_input":"2025-08-16T05:18:15.642705Z","iopub.status.idle":"2025-08-16T05:18:42.781189Z","shell.execute_reply.started":"2025-08-16T05:18:15.642655Z","shell.execute_reply":"2025-08-16T05:18:42.776678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ORIGINAL CATEGORICAL ENCODING - Enhanced\nprint(\"Applying enhanced categorical encoding...\")\ndata_xgb = X.with_columns([\n    (pl.col(c).rank(\"dense\") - 1).fill_null(-1).cast(pl.Int32) \n    for c in cat_features_final\n])\n\n# ORIGINAL DATA SPLITS - Enhanced with validation\nn1 = 16487352  # split train to train and val (10%) in time\nn2 = train.height\ndata_xgb_tr, data_xgb_va, data_xgb_te = data_xgb[:n2], data_xgb[n1:n2], data_xgb[n2:]\ny_tr, y_va, y_te = y[:n2], y[n1:n2], y[n2:]\ngroups_tr, groups_va, groups_te = groups[:n2], groups[n1:n2], groups[n2:]\n\n# Compute group sizes\ngroup_sizes_tr = groups_tr.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_va = groups_va.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\ngroup_sizes_te = groups_te.group_by('ranker_id', maintain_order=True).agg(pl.len())['len'].to_numpy()\n\nprint(f\"Train split: {data_xgb_tr.shape}, Groups: {len(group_sizes_tr)}\")\nprint(f\"Validation split: {data_xgb_va.shape}, Groups: {len(group_sizes_va)}\")\nprint(f\"Test split: {data_xgb_te.shape}, Groups: {len(group_sizes_te)}\")\n\n# Create DMatrix objects\ndtrain = xgb.DMatrix(data_xgb_tr.to_numpy(), label=y_tr.to_numpy().flatten(), group=group_sizes_tr)  ###unquote these two lines if you need training\ndval = xgb.DMatrix(data_xgb_va.to_numpy(), label=y_va.to_numpy().flatten(), group=group_sizes_va)\ndtest = xgb.DMatrix(data_xgb_te.to_numpy(), label=y_te.to_numpy().flatten(), group=group_sizes_te)\n\nprint(\"DMatrix objects created successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:20:17.972242Z","iopub.execute_input":"2025-08-16T05:20:17.972639Z","iopub.status.idle":"2025-08-16T05:20:41.103246Z","shell.execute_reply.started":"2025-08-16T05:20:17.972609Z","shell.execute_reply":"2025-08-16T05:20:41.097443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_params = {\n        'objective': 'rank:pairwise',\n        'eval_metric': 'ndcg@3',\n        \"learning_rate\": 0.022641389657079056,\n        \"max_depth\": 14,\n        \"min_child_weight\": 2,\n        \"subsample\": 0.8842234913702768,\n        \"colsample_bytree\": 0.45840689146263086,\n        \"gamma\": 3.3084297630544888,\n        \"lambda\": 6.952586917313028,\n        \"alpha\": 0.6395254133055179,\n        \n        # NEW: Enhanced parameters\n        \"colsample_bynode\": 0.8,\n        \"colsample_bylevel\": 0.9,\n        \"max_delta_step\": 1,  # Helps with imbalanced ranking\n        \n        'random_state': RANDOM_STATE,\n        'n_jobs': -1,\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T08:42:09.572772Z","iopub.execute_input":"2025-08-15T08:42:09.573132Z","iopub.status.idle":"2025-08-15T08:42:09.584368Z","shell.execute_reply.started":"2025-08-15T08:42:09.573104Z","shell.execute_reply":"2025-08-15T08:42:09.579376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_model = xgb.train(\n        xgb_params,\n        dtrain,\n        num_boost_round=2000,\n        evals=[(dtrain, 'train'), (dval, 'val')],\n        early_stopping_rounds=75,\n        verbose_eval= 50\n    )\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T08:42:12.402863Z","iopub.execute_input":"2025-08-15T08:42:12.403186Z","iopub.status.idle":"2025-08-15T09:18:37.491274Z","shell.execute_reply.started":"2025-08-15T08:42:12.403159Z","shell.execute_reply":"2025-08-15T09:18:37.485878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:25:22.126638Z","iopub.execute_input":"2025-08-16T05:25:22.127021Z","iopub.status.idle":"2025-08-16T05:25:22.136657Z","shell.execute_reply.started":"2025-08-16T05:25:22.126981Z","shell.execute_reply":"2025-08-16T05:25:22.132089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = f'xgb_full_final_model.pkl'\nwith open(model_path, 'wb') as f:\n    pickle.dump(xgb_model, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T09:23:59.864561Z","iopub.execute_input":"2025-08-15T09:23:59.864876Z","iopub.status.idle":"2025-08-15T09:24:00.744835Z","shell.execute_reply.started":"2025-08-15T09:23:59.864851Z","shell.execute_reply":"2025-08-15T09:24:00.737817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_path = '/kaggle/input/single-model-final-submission/xgb_full_model (1).pkl'\n# with open(model_path, 'rb') as f:\n#     xgb_model = pickle.load(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:25:54.454685Z","iopub.execute_input":"2025-08-16T05:25:54.455108Z","iopub.status.idle":"2025-08-16T05:25:55.643944Z","shell.execute_reply.started":"2025-08-16T05:25:54.455053Z","shell.execute_reply":"2025-08-16T05:25:55.637000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ENHANCED EVALUATION with multiple metrics\n# print(\"=== Enhanced Model Evaluation ===\")\n\n# Predictions\n# xgb_tr_preds = xgb_model.predict(dtrain)\n# xgb_va_preds = xgb_model.predict(dval)\nxgb_te_preds = xgb_model.predict(dtest)\n\n# Multiple hitrate metrics\n# y_tr_flat = y_tr.to_numpy().flatten()\n# y_va_flat = y_va.to_numpy().flatten()\ny_te_flat = y_te.to_numpy().flatten()\n# groups_tr_flat = groups_tr.to_numpy().flatten()\n# groups_va_flat = groups_va.to_numpy().flatten()\ngroups_te_flat = groups_te.to_numpy().flatten()\n\n# Calculate multiple k values\n# k_values = [1, 3, 5, 10]\n# print(\"\\nTrain Set Performance:\")\n# for k in k_values:\n#     hr = hitrate_at_k(y_tr_flat, xgb_tr_preds, groups_tr_flat, k)\n#     print(f\"HitRate@{k}: {hr:.6f}\")\n\n# print(\"\\nValidation Set Performance:\")\n# for k in k_values:\n#     hr = hitrate_at_k(y_va_flat, xgb_va_preds, groups_va_flat, k)\n#     print(f\"HitRate@{k}: {hr:.6f}\")\n\n# print(\"\\nTest Set Performance:\")\n# for k in k_values:\n#     hr = hitrate_at_k(y_te_flat, xgb_te_preds, groups_te_flat, k)\n#     print(f\"HitRate@{k}: {hr:.6f}\")\n\n# # Store primary metric\n# xgb_hr3 = hitrate_at_3(y_va_flat, xgb_va_preds, groups_va_flat)\n# print(f\"\\n*** Primary Validation HitRate@3: {xgb_hr3:.6f} ***\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:26:31.843827Z","iopub.execute_input":"2025-08-16T05:26:31.844229Z","iopub.status.idle":"2025-08-16T05:26:43.725146Z","shell.execute_reply.started":"2025-08-16T05:26:31.844199Z","shell.execute_reply":"2025-08-16T05:26:43.720219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ENHANCED FEATURE IMPORTANCE ANALYSIS\n# print(\"\\n=== Enhanced Feature Importance Analysis ===\")\n\n# # Get multiple importance types\n# gain_importance = xgb_model.get_score(importance_type='gain')\n# # freq_importance = xgb_model.get_score(importance_type='frequency')\n# cover_importance = xgb_model.get_score(importance_type='cover')\n\n# # Create comprehensive importance dataframe\n# all_features = set(gain_importance.keys()) | set(cover_importance.keys())\n# importance_data = []\n\n# for feature in all_features:\n#     importance_data.append({\n#         'feature': feature,\n#         'gain': gain_importance.get(feature, 0),\n#         # 'frequency': freq_importance.get(feature, 0),\n#         'cover': cover_importance.get(feature, 0)\n#     })\n\n# importance_df = pl.DataFrame(importance_data).sort('gain', descending=True)\n\n# print(\"\\nTop 30 Features by Gain:\")\n# print(importance_df.head(30).to_pandas().to_string(index=False))\n\n# # Feature categories analysis\n# feature_categories = {\n#     'price': ['price', 'cost', 'fee', 'fare'],\n#     'time': ['time', 'hour', 'day', 'duration', 'date'],\n#     'route': ['route', 'airport', 'carrier', 'flight'],\n#     'ranking': ['rank', 'pct', 'ratio'],\n#     'group': ['group', 'size'],\n#     'service': ['cabin', 'baggage', 'vip', 'class']\n# }\n\n# category_importance = {cat: 0 for cat in feature_categories.keys()}\n# category_importance['other'] = 0\n\n# for row in importance_df.to_dicts():\n#     feature_name = row['feature'].lower()\n#     categorized = False\n    \n#     for category, keywords in feature_categories.items():\n#         if any(keyword in feature_name for keyword in keywords):\n#             category_importance[category] += row['gain']\n#             categorized = True\n#             break\n    \n#     if not categorized:\n#         category_importance['other'] += row['gain']\n\n# print(\"\\nFeature Importance by Category:\")\n# for category, importance in sorted(category_importance.items(), key=lambda x: x[1], reverse=True):\n#     print(f\"{category.capitalize()}: {importance:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T09:25:28.497714Z","iopub.execute_input":"2025-08-15T09:25:28.498073Z","iopub.status.idle":"2025-08-15T09:25:28.720737Z","shell.execute_reply.started":"2025-08-15T09:25:28.498046Z","shell.execute_reply":"2025-08-15T09:25:28.714135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_rank(test: pl.DataFrame, submission_xgb: pl.DataFrame, penalty_factor=0.1):\n    \"\"\"Enhanced re-ranking function with improved flight deduplication\"\"\"\n    \n    # Enhanced flight comparison columns\n    COLS_TO_COMPARE = [\n        \"legs0_departureAt\",\n        \"legs0_arrivalAt\", \n        \"legs1_departureAt\",\n        \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\",\n        \"legs1_segments0_flightNumber\",\n        \"legs0_segments0_aircraft_code\",\n        \"legs1_segments0_aircraft_code\",\n        \"legs0_segments0_departureFrom_airport_iata\",\n        \"legs1_segments0_departureFrom_airport_iata\",\n        \"legs0_segments0_arrivalTo_airport_iata\",\n        \"legs1_segments0_arrivalTo_airport_iata\",\n        \"legs0_segments0_marketingCarrier_code\",\n        \"legs1_segments0_marketingCarrier_code\",\n    ]\n\n    # Ensure columns exist and convert to string\n    available_cols = [c for c in COLS_TO_COMPARE if c in test.columns]\n    test_processed = test.with_columns(\n        [pl.col(c).cast(str).fill_null(\"NULL\") for c in available_cols]\n    )\n\n    df = submission_xgb.join(test_processed, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    # Create comprehensive flight hash\n    if len(available_cols) >= 6:  # Minimum required columns\n        hash_expr = pl.concat_str([pl.col(c) for c in available_cols[:10]], separator=\"_\")\n    else:\n        # Fallback to basic hash\n        hash_expr = (\n            pl.col(\"legs0_departureAt\").cast(str).fill_null(\"NULL\") + \"_\" +\n            pl.col(\"legs0_arrivalAt\").cast(str).fill_null(\"NULL\") + \"_\" +\n            pl.col(\"legs1_departureAt\").cast(str).fill_null(\"NULL\") + \"_\" +\n            pl.col(\"legs1_arrivalAt\").cast(str).fill_null(\"NULL\")\n        )\n    \n    df = df.with_columns(hash_expr.alias(\"flight_hash\"))\n\n    # Enhanced scoring logic\n    df = df.with_columns([\n        pl.max(\"pred_score\").over([\"ranker_id\", \"flight_hash\"]).alias(\"max_score_same_flight\"),\n        pl.count().over([\"ranker_id\", \"flight_hash\"]).alias(\"duplicate_count\")\n    ])\n\n    # Apply penalty with consideration for duplicate count\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\") - \n            penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\")) *\n            pl.col(\"duplicate_count\").clip(upper_bound=5) / 5.0  # Scale penalty by duplicates\n        ).alias(\"reorder_score\")\n    )\n\n    # Re-rank with enhanced score\n    df = df.with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\", \"duplicate_count\"])\n\n# ENHANCED SUBMISSION GENERATION WITH ENSEMBLE\n\n# Use ensemble predictions for submission\nsubmission_xgb = (\n    test.select(['Id', 'ranker_id'])\n    .with_columns(pl.Series('pred_score', xgb_te_preds))  # Using ensemble predictions\n    .with_columns(\n        pl.col('pred_score')\n        .rank(method='ordinal', descending=True)\n        .over('ranker_id')\n        .cast(pl.Int32)\n        .alias('selected')\n    )\n    .select(['Id', 'ranker_id', 'selected', 'pred_score'])\n)\n\nprint(\"Applying enhanced re-ranking...\")\ntop = re_rank(test, submission_xgb)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:26:47.890766Z","iopub.execute_input":"2025-08-16T05:26:47.891112Z","iopub.status.idle":"2025-08-16T05:26:50.792469Z","shell.execute_reply.started":"2025-08-16T05:26:47.891085Z","shell.execute_reply":"2025-08-16T05:26:50.788589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_xgb = (\n    submission_xgb.join(top, on=[\"Id\", \"ranker_id\"], how=\"left\")\n    .with_columns(\n        [\n            pl.when(pl.col(\"new_selected\").is_not_null())\n            .then(pl.col(\"new_selected\"))\n            .otherwise(pl.col(\"selected\"))\n            .alias(\"selected\")\n        ]\n    )\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:27:06.646798Z","iopub.execute_input":"2025-08-16T05:27:06.647167Z","iopub.status.idle":"2025-08-16T05:27:06.898200Z","shell.execute_reply.started":"2025-08-16T05:27:06.647137Z","shell.execute_reply":"2025-08-16T05:27:06.891393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_xgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-16T05:27:09.340506Z","iopub.execute_input":"2025-08-16T05:27:09.340801Z","iopub.status.idle":"2025-08-16T05:27:10.834554Z","shell.execute_reply.started":"2025-08-16T05:27:09.340777Z","shell.execute_reply":"2025-08-16T05:27:10.829222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_xgb.write_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-15T09:28:08.011550Z","iopub.execute_input":"2025-08-15T09:28:08.011918Z","iopub.status.idle":"2025-08-15T09:28:08.894277Z","shell.execute_reply.started":"2025-08-15T09:28:08.011892Z","shell.execute_reply":"2025-08-15T09:28:08.885984Z"}},"outputs":[],"execution_count":null}]}