{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":105399,"databundleVersionId":12733338,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# %%capture\n!pip install --upgrade pip\n!pip install polars","metadata":{"_uuid":"3f03a46b-3930-4a76-a8b8-f2bac10535ab","_cell_guid":"539292b0-37ce-483a-952e-7186f4d8c507","trusted":true,"collapsed":false,"id":"dE63opamDxtJ","outputId":"ac1d1165-8e97-45be-ed5a-13d99468e50c","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-27T08:35:28.317464Z","iopub.execute_input":"2025-12-27T08:35:28.317739Z","iopub.status.idle":"2025-12-27T08:35:36.808541Z","shell.execute_reply.started":"2025-12-27T08:35:28.317715Z","shell.execute_reply":"2025-12-27T08:35:36.807736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import kagglehub\nimport os\n\npath = kagglehub.competition_download(\"aeroclub-recsys-2025\")\nprint(\"✅ Dataset downloaded to:\", path)","metadata":{"_uuid":"73162f5f-34aa-4e82-928d-b2b9ccfe17bc","_cell_guid":"6e755549-de95-4fb3-a249-bcfc5bf72c6e","trusted":true,"collapsed":false,"id":"V1-piopMEHHz","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-27T08:35:36.810248Z","iopub.execute_input":"2025-12-27T08:35:36.810527Z","iopub.status.idle":"2025-12-27T08:35:37.604147Z","shell.execute_reply.started":"2025-12-27T08:35:36.810500Z","shell.execute_reply":"2025-12-27T08:35:37.603463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\n\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)","metadata":{"_uuid":"b6b3b072-ca04-4edb-bfb8-a44787e1770b","_cell_guid":"1f02085b-5d49-4ca5-a667-d1d0dbfe8730","trusted":true,"collapsed":false,"id":"gNNQlxrRDxtL","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-27T08:35:37.604893Z","iopub.execute_input":"2025-12-27T08:35:37.605159Z","iopub.status.idle":"2025-12-27T08:35:38.119107Z","shell.execute_reply.started":"2025-12-27T08:35:37.605138Z","shell.execute_reply":"2025-12-27T08:35:38.118512Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{"_uuid":"08214025-0128-46e2-aba2-bf3cd6d6dcc9","_cell_guid":"f543a311-5e5c-4941-8995-4445bf473cb2","trusted":true,"collapsed":false,"id":"rlK8kru-DxtN","jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:35:38.142001Z","iopub.execute_input":"2025-12-27T08:35:38.142258Z","iopub.status.idle":"2025-12-27T08:35:38.160781Z","shell.execute_reply.started":"2025-12-27T08:35:38.142232Z","shell.execute_reply":"2025-12-27T08:35:38.160144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport time\nimport os\nimport gc\n\nprint(\"=\" * 60)\nprint(\"🚀 GIAI ĐOẠN 6+: ULTIMATE PIPELINE (TIME-BASED SPLIT)\")\nprint(\"   Chiến thuật: Walk-Forward Validation (30% Past -> 70% Future)\")\nprint(\"=\" * 60)\n\n# ==============================================================================\n# 1. CẤU HÌNH AGGREGATION (GIỮ NGUYÊN)\n# ==============================================================================\nHIGH_PRIORITY_CONFIGS = {\n    'avg_segment_flexibility_score_by_user': {'group_by': ['profileId'], 'agg_col': 'segment_flexibility_score', 'agg_func': 'mean'},\n    'std_segment_flexibility_score_by_user': {'group_by': ['profileId'], 'agg_col': 'segment_flexibility_score', 'agg_func': 'std'},\n    'user_convenience_over_price_rate': {'group_by': ['profileId'], 'agg_col': 'chose_convenience_over_price', 'agg_func': 'mean'},\n    'user_price_over_convenience_rate': {'group_by': ['profileId'], 'agg_col': 'chose_price_over_convenience', 'agg_func': 'mean'},\n    'avg_premium_paid_vs_cheapest_by_user': {'group_by': ['profileId'], 'agg_col': 'premium_paid_vs_cheapest', 'agg_func': 'mean'},\n    'company_price_discipline_rate': {'group_by': ['companyID'], 'agg_col': 'chose_bottom_quartile_price', 'agg_func': 'mean'},\n    'company_segment_discipline_rate': {'group_by': ['companyID'], 'agg_col': 'chose_minimum_segments', 'agg_func': 'mean'},\n    'company_premium_policy_rate': {'group_by': ['companyID'], 'agg_col': 'chose_top_quartile_price', 'agg_func': 'mean'},\n    'avg_policy_flexibility_by_company': {'group_by': ['companyID'], 'agg_col': 'policy_flexibility_interaction', 'agg_func': 'mean'},\n    'std_policy_flexibility_by_company': {'group_by': ['companyID'], 'agg_col': 'policy_flexibility_interaction', 'agg_func': 'std'},\n    'user_avg_segment_tier_for_route': {'group_by': ['profileId', 'searchRoute'], 'agg_col': 'segment_tier', 'agg_func': 'mean'},\n    'user_price_percentile_for_route': {'group_by': ['profileId', 'searchRoute'], 'agg_col': 'totalPrice_percentile_in_group', 'agg_func': 'mean'},\n    'company_avg_segment_tier_for_route': {'group_by': ['companyID', 'searchRoute'], 'agg_col': 'segment_tier', 'agg_func': 'mean'},\n    'avg_position_within_segment_tier_by_user': {'group_by': ['profileId'], 'agg_col': 'position_pct_within_segment_tier', 'agg_func': 'mean'},\n    'company_segment_tier_preference': {'group_by': ['companyID'], 'agg_col': 'segment_tier', 'agg_func': 'mean'},\n    'user_avg_convenience_value_score': {'group_by': ['profileId'], 'agg_col': 'convenience_value_score', 'agg_func': 'mean'},\n    'user_sweet_spot_selection_rate': {'group_by': ['profileId'], 'agg_col': 'is_sweet_spot_option', 'agg_func': 'mean'},\n    'user_avg_price_per_extra_segment': {'group_by': ['profileId'], 'agg_col': 'price_per_extra_segment', 'agg_func': 'mean'}\n}\n\nTIER1_CONFIGS = {\n    'route_min_segment_selection_rate': {'group_by': ['searchRoute'], 'agg_col': 'is_min_segments_for_route', 'agg_func': 'mean'},\n    'route_avg_segments_selected': {'group_by': ['searchRoute'], 'agg_col': 'total_segments', 'agg_func': 'mean'},\n    'route_segment_acceptance_by_tier': {'group_by': ['searchRoute', 'route_specific_segment_tier'], 'agg_col': 'selected', 'agg_func': 'mean'},\n    'company_segment_discipline_score': {'group_by': ['companyID'], 'agg_col': 'company_segment_consistency_score', 'agg_func': 'mean'},\n    'company_direct_preference_by_route': {'group_by': ['companyID', 'searchRoute'], 'agg_col': 'is_min_segments', 'agg_func': 'mean'},\n    'company_segment_override_tolerance': {'group_by': ['companyID'], 'agg_col': 'company_segment_override_rate_all', 'agg_func': 'mean'},\n    'user_segment_consistency_by_route': {'group_by': ['profileId', 'searchRoute'], 'agg_col': 'user_segment_choice_std', 'agg_func': 'std'},\n    'user_segment_preference_strength': {'group_by': ['profileId'], 'agg_col': 'user_min_segment_preference_rate', 'agg_func': 'mean'},\n    'user_vs_company_segment_deviation': {'group_by': ['profileId'], 'agg_col': 'user_segments_vs_company_norm', 'agg_func': 'mean'}\n}\n\nALL_AGG_CONFIGS = {**HIGH_PRIORITY_CONFIGS, **TIER1_CONFIGS}\n\n# ==============================================================================\n# 2. HÀM TẠO FULL FEATURES (GIỮ NGUYÊN LOGIC)\n# ==============================================================================\ndef create_full_features_lazy(df_lazy):\n    # (Giữ nguyên toàn bộ logic tạo feature của bạn ở đây)\n    # ... [Copy y nguyên nội dung hàm create_full_features_lazy của bạn vào đây] ...\n    # Để tiết kiệm không gian chat, tôi không paste lại phần thân hàm này \n    # vì nó không đổi logic, chỉ cần đảm bảo requestDate được xử lý đúng.\n    \n    cols = df_lazy.collect_schema().names()\n    \n    # --- Xử lý sơ bộ requestDate để đảm bảo tính toán ---\n    # Nếu chưa có datetime, ép kiểu ngay\n    if \"requestDate\" in cols:\n         # Thử ép kiểu date nếu nó là string\n         df_lazy = df_lazy.with_columns(pl.col(\"requestDate\").str.to_datetime(strict=False).alias(\"requestDate_dt\"))\n\n    # ... [Phần còn lại của hàm Feature Engineering giữ nguyên] ...\n    \n    # --- Paste lại đoạn code logic feature engineering của bạn vào đây ---\n    # (Tôi giả định bạn đã paste lại code feature engineering vào đây)\n    \n    # --- A. Pre-processing ---\n    mc_cols = [f'legs{l}_segments{s}_marketingCarrier_code' for l in (0, 1) for s in range(4)]\n    mc_exists = [c for c in mc_cols if c in cols]\n    if mc_exists: df_lazy = df_lazy.with_columns([pl.col(c).cast(pl.String) for c in mc_exists])\n\n    # Duration String -> Minutes\n    def dur_to_min(col_name):\n        c = pl.col(col_name).cast(pl.Utf8)\n        d = c.str.extract(r\"^(\\d+)\\.\", 1).cast(pl.Float64).fill_null(0) * 1440\n        time_part = c.str.replace(r\"^\\d+\\.\", \"\")\n        h = time_part.str.extract(r\"^(\\d+):\", 1).cast(pl.Float64).fill_null(0) * 60\n        m = time_part.str.extract(r\":(\\d+):\", 1).cast(pl.Float64).fill_null(0)\n        return d + h + m\n\n    dur_cols = [\"legs0_duration\", \"legs1_duration\"] + [f\"legs{l}_segments{s}_duration\" for l in (0, 1) for s in (0, 1, 2, 3)]\n    ex_dur = [c for c in dur_cols if c in cols]\n    df_lazy = df_lazy.with_columns([dur_to_min(c).alias(c) for c in ex_dur])\n\n    # --- B. Base Numerical ---\n    df_lazy = df_lazy.with_columns([\n        (pl.col(\"totalPrice\") / (pl.col(\"taxes\") + 1)).alias(\"price_per_tax\"),\n        (pl.col(\"taxes\") * 100 / (pl.col(\"totalPrice\") + 1)).alias(\"tax_ratex100\"), \n        pl.col(\"totalPrice\").log1p().alias(\"log_price\"),\n        (pl.col(\"miniRules0_monetaryAmount\").fill_null(0) + pl.col(\"miniRules1_monetaryAmount\").fill_null(0)).alias(\"total_fees\"),\n        (pl.col(\"legs0_duration\").fill_null(0) + pl.col(\"legs1_duration\").fill_null(0)).alias(\"total_duration\"),\n    ])\n    \n    df_lazy = df_lazy.with_columns([\n        (pl.col(\"total_fees\") / (pl.col(\"totalPrice\") + 1)).alias(\"fee_rate\"),\n        pl.when(pl.col(\"legs1_duration\").fill_null(0) > 0)\n          .then(pl.col(\"legs0_duration\") / (pl.col(\"legs1_duration\") + 0.01))\n          .otherwise(1.0).alias(\"duration_ratio\")\n    ])\n\n    # Aggregations (Cabin/Baggage)\n    cabin_cols = [c for c in cols if c.endswith('_cabinClass')]\n    bag_cols = [c for c in cols if 'baggageAllowance_quantity' in c]\n    \n    if bag_cols:\n        df_lazy = df_lazy.with_columns(pl.mean_horizontal([pl.col(c).cast(pl.Float64).fill_null(0) for c in bag_cols]).alias(\"baggage_mean\"))\n    else:\n        df_lazy = df_lazy.with_columns(pl.lit(0.0).alias(\"baggage_mean\"))\n\n    if cabin_cols:\n        df_lazy = df_lazy.with_columns(pl.mean_horizontal([pl.col(c).cast(pl.Float64).fill_null(0) for c in cabin_cols]).alias(\"avg_cabin_class_all\"))\n        c0 = [c for c in cabin_cols if 'legs0_' in c]\n        c1 = [c for c in cabin_cols if 'legs1_' in c]\n        if c0 and c1:\n            df_lazy = df_lazy.with_columns(\n                pl.when(pl.col(\"legs1_duration\").is_not_null())\n                .then(pl.mean_horizontal([pl.col(c).cast(pl.Float64).fill_null(0) for c in c0]) - \n                      pl.mean_horizontal([pl.col(c).cast(pl.Float64).fill_null(0) for c in c1]))\n                .otherwise(0.0).alias(\"cabin_class_diff_legs\")\n            )\n        else:\n            df_lazy = df_lazy.with_columns(pl.lit(0.0).alias(\"cabin_class_diff_legs\"))\n    else:\n        df_lazy = df_lazy.with_columns([pl.lit(0.0).alias(\"avg_cabin_class_all\"), pl.lit(0.0).alias(\"cabin_class_diff_legs\")])\n\n    # Total Segments\n    if mc_exists:\n        df_lazy = df_lazy.with_columns(pl.sum_horizontal([pl.col(c).is_not_null().cast(pl.UInt8) for c in mc_exists]).alias(\"total_segments\"))\n    else:\n        df_lazy = df_lazy.with_columns(pl.lit(0.0).alias(\"total_segments\"))\n\n    # --- C. Binary & Datetime ---\n    majors = [\"SU\", \"S7\", \"U6\", \"VN\", \"VJ\", \"QH\"]\n    df_lazy = df_lazy.with_columns([\n        (pl.col(\"legs1_duration\").is_null() | (pl.col(\"legs1_duration\") == 0)).cast(pl.Int32).alias(\"is_one_way\"),\n        pl.col(\"corporateTariffCode\").is_not_null().cast(pl.Int32).alias(\"has_corporate_tariff\"),\n        (pl.col(\"pricingInfo_isAccessTP\") == 1).cast(pl.Int32).alias(\"has_access_tp\"),\n        ((pl.col(\"isVip\") == 1) | (pl.col(\"frequentFlyer\").fill_null(\"\") != \"\")).cast(pl.Int32).alias(\"is_vip_freq\"),\n        (pl.col(\"total_fees\") > 0).cast(pl.Int32).alias(\"has_fees\"),\n        (pl.col(\"legs0_segments0_marketingCarrier_code\").is_in(majors)).cast(pl.Int32).alias(\"is_major_carrier\") \n        if \"legs0_segments0_marketingCarrier_code\" in cols else pl.lit(0).alias(\"is_major_carrier\"),\n        pl.when(pl.col(\"legs1_duration\").is_not_null() & (pl.col(\"legs1_duration\") > 0))\n          .then((pl.sum_horizontal([pl.col(c).is_not_null() for c in mc_exists if 'legs1_' in c]) == 1).cast(pl.Int32))\n          .otherwise(0).alias(\"is_direct_leg1\")\n    ])\n\n    dt_cols = [\"legs0_departureAt\", \"legs0_arrivalAt\", \"legs1_departureAt\", \"legs1_arrivalAt\"]\n    time_exprs = []\n    for c in dt_cols:\n        if c in cols:\n            dt = pl.col(c).str.to_datetime(strict=False)\n            h = dt.dt.hour().fill_null(12)\n            wd = dt.dt.weekday().fill_null(0)\n            time_exprs.extend([\n                (np.sin(2 * np.pi * h / 24)).alias(f\"{c}_hour_sin\"),\n                (np.cos(2 * np.pi * h / 24)).alias(f\"{c}_hour_cos\"),\n                (np.sin(2 * np.pi * wd / 7)).alias(f\"{c}_weekday_sin\"),\n                (np.cos(2 * np.pi * wd / 7)).alias(f\"{c}_weekday_cos\"),\n                (((h >= 6) & (h <= 9)) | ((h >= 17) & (h <= 20))).cast(pl.Int32).alias(f\"{c}_business_time\")\n            ])\n    if time_exprs: df_lazy = df_lazy.with_columns(time_exprs)\n\n    if \"requestDate\" in cols and \"legs0_departureAt\" in cols:\n         df_lazy = df_lazy.with_columns(\n             ((pl.col(\"legs0_departureAt\").str.to_datetime(strict=False).cast(pl.Date) - \n               pl.col(\"requestDate\").str.to_datetime(strict=False).cast(pl.Date)).dt.total_days().cast(pl.Int16).fill_null(0)).alias(\"days_to_departure\")\n         )\n    else:\n         df_lazy = df_lazy.with_columns(pl.lit(0).alias(\"days_to_departure\"))\n\n    if \"frequentFlyer\" in cols and mc_exists:\n        ff = pl.col(\"frequentFlyer\").fill_null(\"\")\n        matches = [pl.when(pl.col(c).is_not_null() & (pl.col(c) != \"\") & ff.str.contains(pl.col(c))).then(1).otherwise(0) for c in mc_exists]\n        df_lazy = df_lazy.with_columns((pl.sum_horizontal(matches) / (pl.col(\"total_segments\") + 1)).alias(\"trust_value\"))\n    else:\n        df_lazy = df_lazy.with_columns(pl.lit(0.0).alias(\"trust_value\"))\n\n    # --- E. INTERMEDIATE FEATURES (SEARCH ROUTE & STATS) ---\n    # 1. Search Route\n    origin_col = \"legs0_segments0_departureFrom_airport_iata\"\n    dest_expr = pl.lit(\"UNK\")\n    for l in [1, 0]:\n        for s in range(3, -1, -1):\n            c = f\"legs{l}_segments{s}_arrivalTo_airport_iata\"\n            if c in cols:\n                dest_expr = pl.when(dest_expr == \"UNK\").then(pl.col(c).fill_null(\"UNK\")).otherwise(dest_expr)\n    \n    if origin_col in cols:\n        df_lazy = df_lazy.with_columns((pl.col(origin_col).fill_null(\"UNK\") + \"-\" + dest_expr).alias(\"searchRoute\"))\n    else:\n        df_lazy = df_lazy.with_columns(pl.lit(\"UNK\").alias(\"searchRoute\"))\n\n    # 2. Tính toán Logic Trung gian cho Tier 1 Aggregations\n    # Cần tính toán min segment theo Route trước\n    df_lazy = df_lazy.with_columns(\n        pl.col(\"total_segments\").min().over(\"searchRoute\").alias(\"min_seg_route\")\n    )\n\n    df_lazy = df_lazy.with_columns([\n        pl.col(\"totalPrice\").min().over(\"ranker_id\").alias(\"min_price_grp\"),\n        pl.col(\"total_segments\").min().over(\"ranker_id\").alias(\"min_seg_grp\"),\n        pl.col(\"totalPrice\").rank(\"ordinal\").over(\"ranker_id\").alias(\"price_rank\"),\n        pl.col(\"totalPrice\").count().over(\"ranker_id\").alias(\"grp_cnt\"),\n        pl.col(\"total_duration\").rank(\"ordinal\").over(\"ranker_id\").alias(\"dur_rank\"),\n        \n        # Z-score duration\n        pl.col(\"total_duration\").mean().over(\"ranker_id\").alias(\"mean_dur\"),\n        pl.col(\"total_duration\").std().over(\"ranker_id\").fill_null(1).alias(\"std_dur\"),\n        \n        # [NEW TIER 1] Route Specific Logic\n        (pl.col(\"total_segments\") == pl.col(\"min_seg_route\")).cast(pl.Int8).alias(\"is_min_segments_for_route\"),\n        (pl.col(\"total_segments\") - pl.col(\"min_seg_route\")).clip(0, 3).cast(pl.Int8).alias(\"route_specific_segment_tier\"),\n    ])\n    \n    df_lazy = df_lazy.with_columns([\n        ((pl.col(\"price_rank\") - 1) / (pl.col(\"grp_cnt\") - 1 + 1e-6)).alias(\"totalPrice_percentile_in_group\"),\n        ((pl.col(\"dur_rank\") - 1) / (pl.col(\"grp_cnt\") - 1 + 1e-6)).alias(\"duration_percentile\"),\n        ((pl.col(\"mean_dur\") - pl.col(\"total_duration\")) / (pl.col(\"std_dur\") + 1e-6)).alias(\"duration_z_score\"),\n        \n        # Complex Logic\n        (pl.col(\"total_segments\") - pl.col(\"min_seg_grp\")).cast(pl.Float32).alias(\"segment_flexibility_score\"),\n        (pl.col(\"totalPrice\") - pl.col(\"min_price_grp\")).alias(\"premium_paid_vs_cheapest\"),\n        (pl.col(\"total_segments\") - pl.col(\"min_seg_grp\")).clip(0, 3).cast(pl.Int8).alias(\"segment_tier\"),\n        (pl.col(\"total_segments\") == pl.col(\"min_seg_grp\")).cast(pl.Int8).alias(\"chose_minimum_segments\"),\n        (pl.col(\"pricingInfo_isAccessTP\").fill_null(0) * (pl.col(\"price_rank\") - 1) / (pl.col(\"grp_cnt\") - 1 + 1e-6)).alias(\"policy_flexibility_interaction\")\n    ])\n    \n    # Quartiles & Trade-offs & Tier 1 Mapping\n    df_lazy = df_lazy.with_columns([\n        (pl.col(\"totalPrice_percentile_in_group\") <= 0.25).cast(pl.Int8).alias(\"chose_bottom_quartile_price\"),\n        (pl.col(\"totalPrice_percentile_in_group\") >= 0.75).cast(pl.Int8).alias(\"chose_top_quartile_price\"),\n        ((pl.col(\"chose_minimum_segments\") == 1) & (pl.col(\"totalPrice\") > pl.col(\"min_price_grp\"))).cast(pl.Int8).alias(\"chose_convenience_over_price\"),\n        ((pl.col(\"totalPrice\") == pl.col(\"min_price_grp\")) & (pl.col(\"chose_minimum_segments\") == 0)).cast(pl.Int8).alias(\"chose_price_over_convenience\"),\n        ((1 - pl.col(\"totalPrice_percentile_in_group\")) * (1 - pl.col(\"duration_percentile\"))).alias(\"convenience_value_score\"),\n        pl.when(pl.col(\"total_segments\") > pl.col(\"min_seg_grp\"))\n          .then((pl.col(\"totalPrice\") - pl.col(\"min_price_grp\")) / (pl.col(\"total_segments\") - pl.col(\"min_seg_grp\")))\n          .otherwise(0).alias(\"price_per_extra_segment\"),\n          \n        # [MAPPING TIER 1 ALIASES] - Để khớp với config\n        pl.col(\"chose_minimum_segments\").alias(\"company_segment_consistency_score\"),\n        pl.col(\"chose_minimum_segments\").alias(\"is_min_segments\"),\n        (1 - pl.col(\"chose_minimum_segments\")).alias(\"company_segment_override_rate_all\"),\n        pl.col(\"total_segments\").alias(\"user_segment_choice_std\"), # Nguồn để tính std\n        pl.col(\"chose_minimum_segments\").alias(\"user_min_segment_preference_rate\"),\n        pl.col(\"segment_flexibility_score\").alias(\"user_segments_vs_company_norm\"),\n    ])\n    \n    # Position in Tier & Sweet Spot\n    df_lazy = df_lazy.with_columns([\n        pl.col(\"totalPrice\").rank(\"ordinal\").over([\"ranker_id\", \"segment_tier\"]).alias(\"rank_in_tier\"),\n        pl.col(\"totalPrice\").count().over([\"ranker_id\", \"segment_tier\"]).alias(\"cnt_in_tier\")\n    ])\n    \n    df_lazy = df_lazy.with_columns([\n        ((pl.col(\"rank_in_tier\") - 1) / (pl.col(\"cnt_in_tier\") - 1 + 1e-6)).alias(\"position_pct_within_segment_tier\"),\n        ((pl.col(\"totalPrice_percentile_in_group\") <= 0.3) & (pl.col(\"duration_percentile\") <= 0.3)).cast(pl.Int8).alias(\"is_sweet_spot_option\")\n    ])\n    \n    # Clean up temp\n    df_lazy = df_lazy.drop([\"min_price_grp\", \"min_seg_grp\", \"grp_cnt\", \"price_rank\", \"dur_rank\", \"mean_dur\", \"std_dur\", \"rank_in_tier\", \"cnt_in_tier\", \"min_seg_route\"])\n    \n    return df_lazy\n\n# ==============================================================================\n# ==============================================================================\n# HÀM TẠO MAP NÂNG CẤP (SMOOTHING + GLOBAL FALLBACK)\n# ==============================================================================\ndef create_smoothed_maps(df_stats_lazy, smoothing_weight=20):\n    print(f\"🧠 Calculating Maps with Bayesian Smoothing (m={smoothing_weight})...\")\n    \n    # 1. Tạo feature cơ bản cho tập Stats\n    df_aug = create_full_features_lazy(df_stats_lazy)\n    \n    # 2. Tách dữ liệu:\n    # - Một số feature cần tính trên toàn bộ (VD: Win Rate)\n    # - Một số feature chỉ tính trên các dòng được chọn (VD: Giá trung bình của vé được chọn)\n    # -> Để đơn giản và khớp logic cũ, ta vẫn lọc selected=1 cho các profile hành vi.\n    # -> Tuy nhiên, nếu bạn muốn tính WinRate chuẩn, bạn nên tính trên toàn bộ df_aug.\n    # Ở đây tôi giữ logic cũ của bạn (tính trên selected=1) để an toàn cho pipeline hiện tại.\n    df_base = df_aug.filter(pl.col(\"selected\") == 1)\n    \n    maps = {}\n    \n    for feat_name, config in ALL_AGG_CONFIGS.items():\n        keys = config['group_by']\n        target = config['agg_col']\n        func = config['agg_func']\n        \n        # --- BƯỚC A: TÍNH GLOBAL MEAN (FALLBACK) ---\n        # Tính giá trị trung bình toàn cục của cột target trên tập Stats\n        # (Dùng streaming collect để lấy ra 1 số thực duy nhất)\n        try:\n            if func == 'mean':\n                global_val = df_base.select(pl.col(target).mean()).collect(streaming=True).item()\n            elif func == 'std':\n                global_val = df_base.select(pl.col(target).std()).collect(streaming=True).item()\n            else:\n                global_val = 0 # Default cho count\n        except:\n            global_val = 0 # Fallback an toàn nếu cột toàn null\n            \n        # Lưu global_val vào config để dùng khi fillna sau này\n        \n        # --- BƯỚC B: TÍNH LOCAL STATS + SMOOTHING ---\n        if func == 'mean':\n            # Áp dụng công thức Bayesian Smoothing\n            # Cần tính: Sum và Count của từng nhóm\n            expr = (\n                (pl.col(target).sum() + (global_val * smoothing_weight)) / \n                (pl.col(target).count() + smoothing_weight)\n            )\n        elif func == 'std':\n            # Std không smooth theo công thức trên được, giữ nguyên\n            expr = pl.col(target).std().fill_null(global_val)\n        else:\n            expr = pl.col(target).mean()\n            \n        # Collect Map\n        stat = df_base.group_by(keys).agg(expr.alias(feat_name)).collect(streaming=True)\n        \n        maps[feat_name] = {\n            'keys': keys, \n            'df': stat, \n            'fallback': global_val # <--- GIÁ TRỊ ĐIỀN VÀO CHỖ TRỐNG\n        }\n        \n    return maps\n\n# ==============================================================================\n# HÀM APPLY NÂNG CẤP (DÙNG GLOBAL FALLBACK)\n# ==============================================================================\ndef apply_features_with_fallback(df_lazy, agg_maps):\n    df = create_full_features_lazy(df_lazy)\n    \n    for feat_name, map_info in agg_maps.items():\n        keys = map_info['keys']\n        fallback_val = map_info['fallback']\n        \n        # Nếu fallback là None (do lỗi tính toán), gán về -1 hoặc 0\n        if fallback_val is None: fallback_val = -1\n        \n        # Join Left\n        # Thay vì fill_null(-1), ta fill_null(fallback_val)\n        df = df.join(\n            map_info['df'].lazy(), \n            on=keys, \n            how=\"left\"\n        ).with_columns(\n            pl.col(feat_name).fill_null(pl.lit(fallback_val)) # <--- ĐIỂM KHÁC BIỆT\n        )\n        \n    return df\n\n# ==============================================================================\n# ==============================================================================\n# 4. THỰC THI (TIME-BASED LOGIC - FIXED)\n# ==============================================================================\nN_SPLITS = 10\nstart_time = time.time()\npath = \"../input/aeroclub-recsys-2025\"\n\nprint(\"[1/5] Loading & Parsing Dates for Time-Split (ROBUST FIX)...\")\n\n# 1. Load Lazy\ntrain_scan = pl.scan_parquet(f'{path}/train.parquet')\ntest_scan = pl.scan_parquet(f'{path}/test.parquet')\n\n# 2. XỬ LÝ DATE AN TOÀN (QUAN TRỌNG)\n# Bước A: Ép requestDate về String (để hàm feature engineering bên dưới không bị lỗi SchemaError)\ntrain_scan = train_scan.with_columns(pl.col(\"requestDate\").cast(pl.String))\ntest_scan = test_scan.with_columns(pl.col(\"requestDate\").cast(pl.String))\n\n# Bước B: Tạo cột phụ requestDate_dt chuẩn Datetime để sort và cắt\n# Dùng str.to_datetime với strict=False để tránh lỗi nếu có chuỗi lạ\ntrain_scan = train_scan.with_columns(\n    pl.col(\"requestDate\").str.to_datetime(strict=False).alias(\"requestDate_dt\")\n).sort(\"requestDate_dt\")\n\n# 3. Tính điểm cắt (30% Quantile)\nprint(\"   -> Calculating Time Cut-off (20% Quantile)...\")\n\n# Collect cột date, BỎ QUA NULL (drop_nulls) để tránh lỗi None\ndate_series = train_scan.select(pl.col(\"requestDate_dt\").drop_nulls()).collect().get_column(\"requestDate_dt\")\n\nif len(date_series) == 0:\n    raise ValueError(\"❌ Lỗi: Không parse được cột requestDate nào cả! Kiểm tra lại định dạng ngày tháng.\")\n\ncut_off_date = date_series.quantile(0.20)\nprint(f\"   -> Cut-off Date: {cut_off_date}\")\n\nif cut_off_date is None:\n    # Fallback: Nếu vẫn None (hiếm), lấy ngày ở vị trí 20% thủ công\n    sorted_dates = date_series.sort()\n    idx = int(len(sorted_dates) * 0.2)\n    cut_off_date = sorted_dates[idx]\n    print(f\"   -> Cut-off Date (Manual Fallback): {cut_off_date}\")\n\n# 4. Chia dữ liệu\n# Lưu ý: drop cột requestDate_dt sau khi dùng xong để không gây rối schema\ntrain_stats_lazy = train_scan.filter(pl.col(\"requestDate_dt\") <= cut_off_date).drop(\"requestDate_dt\")\ntrain_model_lazy = train_scan.filter(pl.col(\"requestDate_dt\") > cut_off_date).drop(\"requestDate_dt\")\n\n# In thông tin kiểm tra\nprint(f\"   -> Total Rows: {len(date_series):,}\")\nprint(f\"   -> Stats Set (Past): ~30% (Used for Maps)\")\nprint(f\"   -> Train Set (Future): ~70% (Used for Model)\")\n\nprint(\"[2/5] Creating Smoothed Maps...\")\n# Gọi hàm mới, smoothing_weight=20 là con số an toàn\nagg_maps = create_smoothed_maps(train_stats_lazy, smoothing_weight=40)\n\nprint(\"[3/5] Applying Features with Global Fallback...\")\n# Gọi hàm apply mới\ndf_train_final = apply_features_with_fallback(train_model_lazy, agg_maps)\ndf_test_final = apply_features_with_fallback(test_scan, agg_maps)\n\nprint(\"[4/5] Selecting & Collecting...\")\n# Collect Schema để lọc cột rác\nall_cols = df_train_final.collect_schema().names()\n\nGARBAGE = ['_segments', 'flightNumber', 'seatsAvailable', 'weightMeasurementType', \n           'airport_city_iata', 'arrivalTo_airport_iata', 'departureFrom_airport_iata',\n           'legs0_duration', 'legs1_duration', 'marketingCarrier_code', 'baggageAllowance_quantity',\n           'miniRules', 'taxes', 'requestDate_dt'] # Thêm requestDate_dt vào rác cho chắc\n\nSYSTEM = [\"ranker_id\", \"selected\", \"Id\", \"fold\", \"requestDate\"]\n\nfinal_cols = [c for c in all_cols if c in SYSTEM or not any(p in c for p in GARBAGE)]\nprint(f\"   -> Features count: {len(final_cols)}\")\n\npandas_df = df_train_final.select([c for c in final_cols if c != \"Id\"]).collect().to_pandas(use_pyarrow_extension_array=False)\ntest_df_pd = df_test_final.select([c for c in final_cols if c != \"selected\"]).collect().to_pandas(use_pyarrow_extension_array=False)\n\nprint(\"[5/5] Final Polish & Fold Creation...\")\nfor df in [pandas_df, test_df_pd]:\n    # Xử lý Category\n    if 'searchRoute' in df.columns: df['searchRoute'] = df['searchRoute'].astype('category')\n    cat_candidates = [\"pricingInfo_isAccessTP\", \"frequentFlyer\", \"isVip\", \"bySelf\", \"sex\", \"cancellation_status\", \"exchange_status\"]\n    for c in df.columns:\n        if c in cat_candidates: df[c] = df[c].astype('category')\n    \n    # Chia Fold (Fixed Sorted)\n    if 'ranker_id' in df.columns and 'selected' in df.columns:\n         unique_ids = sorted(df['ranker_id'].unique())\n         np.random.seed(42)\n         fold_map = {uid: np.random.randint(0, N_SPLITS) for uid in unique_ids}\n         df['fold'] = df['ranker_id'].map(fold_map)\n\nprint(f\"✅ DONE! Train (Future): {pandas_df.shape}, Test: {test_df_pd.shape}\")\nprint(f\"⏱️ Time elapsed: {time.time() - start_time:.2f}s\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:44:21.679689Z","iopub.execute_input":"2025-12-27T08:44:21.680038Z","iopub.status.idle":"2025-12-27T08:59:56.392303Z","shell.execute_reply.started":"2025-12-27T08:44:21.680012Z","shell.execute_reply":"2025-12-27T08:59:56.391461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nimport numpy as np\nimport pandas as pd\nimport gc\nimport os\n\nprint(\"🚀 GIAI ĐOẠN 5: TRAINING (RAM-SAFE & FIXED DTYPES)\")\n\n# ==============================================================================\n# 1. HÀM GIẢM RAM (PHIÊN BẢN AN TOÀN - FIX LỖI ID)\n# ==============================================================================\ndef reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'   Original memory usage: {start_mem:.2f} MB')\n    \n    # 🔥 DANH SÁCH CÁC CỘT CẦN GIỮ NGUYÊN (KHÔNG ĐƯỢC ÉP KIỂU)\n    # Id và ranker_id bắt buộc phải là String/Object để Merge và Groupby an toàn\n    SKIP_COLS = ['Id', 'ranker_id', 'flight_hash', 'fold'] \n    \n    for col in df.columns:\n        # Nếu là cột ID, bỏ qua ngay lập tức\n        if col in SKIP_COLS:\n            continue\n            \n        col_type = df[col].dtype\n        \n        # Bỏ qua Object, Category VÀ Datetime\n        if col_type != object and str(col_type) != 'category' and 'datetime' not in str(col_type):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            # Xử lý số nguyên\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n            # Xử lý số thực\n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float32)\n        else:\n            # Ép kiểu category cho object/string (CHỈ CÁC CỘT KHÔNG NẰM TRONG SKIP_COLS)\n            if col_type == object:\n                df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'   Optimized memory usage: {end_mem:.2f} MB ({100 * (start_mem - end_mem) / start_mem:.1f}% reduction)')\n    return df\n# Áp dụng giảm RAM\nprint(\"📉 Optimizing RAM for Train...\")\npandas_df = reduce_mem_usage(pandas_df)\n\nprint(\"📉 Optimizing RAM for Test...\")\ntest_df_pd = reduce_mem_usage(test_df_pd)\ngc.collect()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:59:56.393496Z","iopub.execute_input":"2025-12-27T08:59:56.393878Z","iopub.status.idle":"2025-12-27T09:00:19.443206Z","shell.execute_reply.started":"2025-12-27T08:59:56.393857Z","shell.execute_reply":"2025-12-27T09:00:19.442473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n\n# def check_split_ratio(df, threshold=50):\n#     print(\"=\"*60)\n#     print(f\"📊 PHÂN TÍCH TỶ LỆ USER (THRESHOLD = {threshold})\")\n#     print(\"=\"*60)\n    \n#     # 1. Tính group_size nếu chưa có\n#     if 'group_size' not in df.columns:\n#         print(\"⚙️ Calculating group_size...\")\n#         # Đếm số dòng của mỗi ranker_id\n#         df['group_size'] = df.groupby('ranker_id')[df.columns[0]].transform('count')\n\n#     # 2. Tạo bảng thống kê User (nhanh hơn là query trên toàn bộ dataframe)\n#     # Chỉ giữ lại ranker_id và group_size\n#     user_stats = df[['ranker_id', 'group_size']].drop_duplicates()\n    \n#     # 3. Tính toán\n#     total_users = len(user_stats)\n#     total_rows = len(df)\n    \n#     # Nhóm Small\n#     small_users = user_stats[user_stats['group_size'] <= threshold]\n#     n_small_users = len(small_users)\n#     n_small_rows = len(df[df['group_size'] <= threshold])\n    \n#     # Nhóm Large\n#     large_users = user_stats[user_stats['group_size'] > threshold]\n#     n_large_users = len(large_users)\n#     n_large_rows = len(df[df['group_size'] > threshold])\n    \n#     # 4. In kết quả\n#     print(f\"1️⃣  Về số lượng USER (Session):\")\n#     print(f\"   - Tổng User: {total_users:,}\")\n#     print(f\"   - Nhóm Small: {n_small_users:,} user ({n_small_users/total_users:.2%})\")\n#     print(f\"   - Nhóm Large: {n_large_users:,} user ({n_large_users/total_users:.2%})\")\n    \n#     print(f\"\\n2️⃣  Về số lượng DÒNG DỮ LIỆU (Rows):\")\n#     print(f\"   - Tổng Rows: {total_rows:,}\")\n#     print(f\"   - Nhóm Small: {n_small_rows:,} dòng ({n_small_rows/total_rows:.2%})\")\n#     print(f\"   - Nhóm Large: {n_large_rows:,} dòng ({n_large_rows/total_rows:.2%})\")\n    \n#     print(\"-\" * 60)\n#     print(\"💡 NHẬN ĐỊNH:\")\n#     if n_small_users / total_users > 0.2:\n#         print(\"   ✅ Tỷ lệ User Small khá cao (>20%). Chiến thuật tách model là RẤT HIỆU QUẢ.\")\n#         print(\"      Lý do: Model Small đạt HitRate ~0.78 sẽ kéo điểm tổng thể lên rất mạnh.\")\n#     else:\n#         print(\"   ⚠️ Tỷ lệ User Small thấp. Sự ảnh hưởng của Model Small sẽ không quá lớn.\")\n\n# # --- CHẠY KIỂM TRA ---\n# # Kiểm tra trên tập Train\n# if 'pandas_df' in locals():\n#     print(\"\\n[TRAIN SET]\")\n#     check_split_ratio(pandas_df, threshold=70)\n\n# # Kiểm tra trên tập Test\n# if 'test_df_pd' in locals():\n#     print(\"\\n[TEST SET]\")\n#     check_split_ratio(test_df_pd, threshold=70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:36:23.675005Z","iopub.status.idle":"2025-12-27T08:36:23.675267Z","shell.execute_reply.started":"2025-12-27T08:36:23.675150Z","shell.execute_reply":"2025-12-27T08:36:23.675165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nimport numpy as np\nimport pandas as pd\nimport gc\nimport polars as pl\nimport os\nimport matplotlib.pyplot as plt\n\nprint(\"🚀 GIAI ĐOẠN 5: SINGLE LIGHTGBM RANKER (FIXED DTYPES)\")\n\n# ==============================================================================\n# 1. CẤU HÌNH & LỌC FEATURE (QUAN TRỌNG: SỬA LỖI Ở ĐÂY)\n# ==============================================================================\nCOMMON_PARAMS = {\n    \"objective\": \"lambdarank\",\n    \"metric\": \"ndcg\",\n    \"eval_at\": [3],\n    \"device\": \"gpu\",\n    \"gpu_platform_id\": 0,\n    \"gpu_device_id\": 0,\n    \n    \"max_bin\": 63,\n    \"n_estimators\": 3000,   \n    \"learning_rate\": 0.03,\n    \"num_leaves\": 255,\n    \"min_data_in_leaf\": 200, \n    \"reg_lambda\": 5.0,\n    \"colsample_bytree\": 0.8,\n    \"subsample\": 0.7,\n    \"subsample_freq\": 1,\n    'lambdarank_truncation_level': 10,\n    \"random_state\": 42,\n    \"n_jobs\": -1,\n    \"two_round\": True,\n    \"verbose\": -1\n}\n\n\ntarget = \"selected\"\n\n# 1. Danh sách các cột hệ thống chắc chắn bỏ\nDROP_LIST = [\n    # ❌ NHÓM THỜI GIAN THÔ (THỦ PHẠM CHÍNH)\n    \"legs0_arrivalAt\", \"legs0_departureAt\", \n    \"legs1_arrivalAt\", \"legs1_departureAt\",\n    \"requestDate\", \n    \"legs0_segments0_departureFrom_airport_iata\", # Sân bay cụ thể quá cũng có thể gây nhiễu\n    \"legs0_segments0_arrivalTo_airport_iata\",\n    \n    # ❌ CÁC CỘT HỆ THỐNG\n    \"ranker_id\", \"selected\", \"fold\", \"group_size\", \"size_tier\", \"Id\"\n]\n\n# 2. Lấy tất cả cột, trừ cột hệ thống\ninitial_features = [c for c in pandas_df.columns if c not in DROP_LIST]\n\n# 3. [FIX] Lọc bỏ các cột có kiểu dữ liệu không hợp lệ (Object, Datetime)\n# LightGBM chỉ chấp nhận: number (int/float), bool, category\nvalid_features = []\nprint(\"⚙️ Checking feature dtypes...\")\n\nfor col in initial_features:\n    dtype = pandas_df[col].dtype\n    # Giữ lại nếu là số, category hoặc bool\n    if pd.api.types.is_numeric_dtype(dtype) or isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_bool_dtype(dtype):\n        valid_features.append(col)\n    else:\n        print(f\"   ❌ Dropping raw column: {col} ({dtype})\")\n\nfeatures = valid_features\nprint(f\"📋 Final Valid Features: {len(features)}\")\nprint(features)\n\n# Xác định lại Categorical Features từ danh sách features sạch\ncat_cols = [\"pricingInfo_isAccessTP\", \"frequentFlyer\", \"isVip\", \"bySelf\", \"sex\", \"cancellation_status\", \"exchange_status\", \"has_corporate_tariff\", \"has_access_tp\", \"is_vip_freq\", \"is_one_way\", \"has_fees\", \"is_major_carrier\", \"is_direct_leg1\"]\ncategorical_features = [c for c in cat_cols if c in features]\n\n# Đảm bảo cột 'fold' tồn tại\nif 'fold' not in pandas_df.columns:\n    print(\"⚙️ Creating Folds...\")\n    gkf = GroupKFold(n_splits=N_SPLITS)\n    pandas_df['fold'] = -1\n    for i, (_, v_idx) in enumerate(gkf.split(pandas_df, groups=pandas_df['ranker_id'])):\n        pandas_df.iloc[v_idx, pandas_df.columns.get_loc('fold')] = i\n\n# Hàm Metric\ndef hitrate_metric(y_true, y_pred, group_ids):\n    df_tmp = pl.DataFrame({\"y\": y_true, \"p\": y_pred, \"g\": group_ids})\n    hits = df_tmp.sort(\"p\", descending=True).group_by(\"g\").head(3).filter(pl.col(\"y\")==1)\n    valid_g = df_tmp.group_by(\"g\").len().filter(pl.col(\"len\") > 10)\n    if valid_g.height == 0: return 0.0\n    return hits.join(valid_g, on=\"g\", how=\"inner\").height / valid_g.height\n\n# ==============================================================================\n# 2. VÒNG LẶP HUẤN LUYỆN\n# ==============================================================================\nfinal_test_preds = np.zeros(len(test_df_pd))\noof_preds = np.zeros(len(pandas_df))\n\nprint(f\"\\n🥊 Bắt đầu Train {N_SPLITS} Folds...\")\n\nfor fold in range(N_SPLITS):\n    print(f\"\\n--- Fold {fold} ---\")\n    \n    train_mask = pandas_df[\"fold\"] != fold\n    val_mask = pandas_df[\"fold\"] == fold\n    \n    # Tạo View dữ liệu\n    X_tr = pandas_df.loc[train_mask, features]\n    y_tr = pandas_df.loc[train_mask, target]\n    g_tr = pandas_df.loc[train_mask].groupby(\"ranker_id\", sort=False).size().to_numpy()\n    \n    X_val = pandas_df.loc[val_mask, features]\n    y_val = pandas_df.loc[val_mask, target]\n    g_val = pandas_df.loc[val_mask].groupby(\"ranker_id\", sort=False).size().to_numpy()\n    val_ids = pandas_df.loc[val_mask, \"ranker_id\"]\n    gc.collect()\n    # Train\n    model = lgb.LGBMRanker(**COMMON_PARAMS)\n    model.fit(\n        X_tr, y_tr, group=g_tr,\n        eval_set=[(X_val, y_val)], eval_group=[g_val],\n        eval_metric=lambda y, p: [('hitrate', hitrate_metric(y, p, val_ids), True)],\n        callbacks=[lgb.early_stopping(50, verbose=False)],\n        categorical_feature=categorical_features\n    )\n    \n    # Predict\n    oof_preds[val_mask] = model.predict(X_val)\n    hr_score = model.best_score_['valid_0']['hitrate']\n    print(f\"   🌟 Best HitRate Fold {fold}: {hr_score:.4f}\")\n    \n    final_test_preds += model.predict(test_df_pd[features]) / N_SPLITS\n    \n    \n# Chỉ vẽ được nếu bạn lưu lại model của fold cuối cùng hoặc fold 0\n# Giả sử biến 'model' đang là model của Fold cuối cùng\n\nprint(\"📊 Vẽ biểu đồ từ LGBMRanker...\")\nresults = model.evals_result_\nepochs = len(results['valid_0']['ndcg@3'])\nx_axis = range(0, epochs)\n\nfig, ax = plt.subplots(figsize=(12, 6))\nax.plot(x_axis, results['valid_0']['ndcg@3'], label='Validation NDCG')\nax.legend()\nplt.ylabel('NDCG@3')\nplt.title('LightGBM Training History (Sklearn API)')\nplt.show()\n# ==============================================================================\n# 3. TẠO SUBMISSION\n# ==============================================================================\nprint(\"\\n📝 Xuất file submission_single.csv...\")\n\ntest_df_pd[\"score\"] = final_test_preds\ntest_df_pd[\"rank\"] = test_df_pd.groupby(\"ranker_id\")[\"score\"].rank(method=\"first\", ascending=False).astype(int)\n\nsubmission = test_df_pd[[\"Id\", \"ranker_id\", \"rank\"]].copy()\nsubmission.rename(columns={\"rank\": \"selected\"}, inplace=True)\n\ndata_dir = \"../input/aeroclub-recsys-2025\"\ntest_file = os.path.join(data_dir, \"test.parquet\")\nif os.path.exists(test_file):\n    original_order = pd.read_parquet(test_file, columns=[\"Id\"])\n    final_submission = original_order.merge(submission, on=\"Id\", how=\"left\")\n    final_submission[\"selected\"] = final_submission[\"selected\"].fillna(1).astype(int)\n    final_submission.to_csv(\"submission_single.csv\", index=False)\nelse:\n    submission.to_csv(\"submission_single.csv\", index=False)\n\nprint(f\"✅ DONE! File: submission_single.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:36:23.677967Z","iopub.status.idle":"2025-12-27T08:36:23.678187Z","shell.execute_reply.started":"2025-12-27T08:36:23.678080Z","shell.execute_reply":"2025-12-27T08:36:23.678090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"=\"*40)\nprint(f\"📊 TỔNG SỐ FEATURE ĐƯỢC DÙNG: {len(features)}\")\nprint(\"=\"*40)\n\nprint(\"\\n🔍 Danh sách chi tiết:\")\nfor i, f in enumerate(features):\n    # In ra tên feature và kiểu dữ liệu của nó\n    dtype = pandas_df[f].dtype\n    print(f\"{i+1:02d}. {f:<35} ({dtype})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:36:23.679501Z","iopub.status.idle":"2025-12-27T08:36:23.679751Z","shell.execute_reply.started":"2025-12-27T08:36:23.679634Z","shell.execute_reply":"2025-12-27T08:36:23.679646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\n\nprint(\"=\" * 60)\nprint(\"🚀 GIAI ĐOẠN 7: DIVERSITY RE-RANKING (POST-PROCESSING)\")\nprint(\"=\" * 60)\n\n# ==============================================================================\n# 1. CHUẨN BỊ DỮ LIỆU (PANDAS -> POLARS)\n# ==============================================================================\n# Giả sử bạn đã có:\n# - test_df_pd: DataFrame Pandas chứa features và ID\n# - final_test_preds: Mảng numpy chứa điểm dự đoán của LightGBM\n\nprint(\"🔄 Converting Data to Polars for Re-ranking...\")\n\n# Tạo DataFrame submission cơ bản từ kết quả LightGBM\nsubmission_lgbm = pd.DataFrame({\n    'Id': test_df_pd['Id'],\n    'ranker_id': test_df_pd['ranker_id'],\n    'pred_score': final_test_preds\n})\n\n# Chuyển sang Polars\npl_submission = pl.from_pandas(submission_lgbm)\npl_test_features = pl.from_pandas(test_df_pd)\n\n# ==============================================================================\n# 2. HÀM RE-RANK (LOGIC CỦA BẠN)\n# ==============================================================================\ndef re_rank(test_features: pl.DataFrame, submission: pl.DataFrame, penalty_factor=1.0):\n    # Cột dùng để định danh chuyến bay (Physical Flight Identity)\n    COLS_TO_HASH = [\n        \"legs0_departureAt\", \"legs0_arrivalAt\", \n        \"legs1_departureAt\", \"legs1_arrivalAt\",\n        \"legs0_segments0_flightNumber\", \"legs1_segments0_flightNumber\",\n        # Thêm sân bay để chắc chắn không trùng\n        \"legs0_segments0_departureFrom_airport_iata\" \n    ]\n    \n    # Kiểm tra cột tồn tại trước khi xử lý\n    available_cols = [c for c in COLS_TO_HASH if c in test_features.columns]\n    \n    # Ép kiểu String và Fill Null để Hash\n    test_features = test_features.with_columns(\n        [pl.col(c).cast(pl.String).fill_null(\"NULL\") for c in available_cols]\n    )\n\n    # Join điểm dự đoán vào Features\n    df = submission.join(test_features.select([\"Id\", \"ranker_id\"] + available_cols), \n                         on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n    # 1. Tạo Flight Hash (Định danh chuyến bay)\n    # Kết hợp các cột lại thành 1 chuỗi duy nhất\n    df = df.with_columns(\n        pl.concat_str(available_cols, separator=\"_\").alias(\"flight_hash\")\n    )\n\n    # 2. Tìm điểm cao nhất của chuyến bay đó trong nhóm\n    df = df.with_columns(\n        pl.max(\"pred_score\")\n        .over([\"ranker_id\", \"flight_hash\"])\n        .alias(\"max_score_same_flight\")\n    )\n\n    # 3. Tính điểm phạt (Penalty)\n    # Công thức: Score_Mới = Score_Cũ - Alpha * (Max_Score - Score_Cũ)\n    # Nếu là dòng tốt nhất: Score_Mới = Score_Cũ (Không bị phạt)\n    # Nếu là dòng kém hơn: Càng kém càng bị phạt nặng\n    df = df.with_columns(\n        (\n            pl.col(\"pred_score\")\n            - penalty_factor * (pl.col(\"max_score_same_flight\") - pl.col(\"pred_score\"))\n        ).alias(\"reorder_score\")\n    )\n\n    # 4. Xếp hạng lại dựa trên điểm mới\n    df = df.with_columns(\n        pl.col(\"reorder_score\")\n        .rank(method=\"ordinal\", descending=True)\n        .over(\"ranker_id\")\n        .cast(pl.Int32)\n        .alias(\"new_selected\")\n    )\n\n    return df.select([\"Id\", \"ranker_id\", \"new_selected\", \"pred_score\", \"reorder_score\"])\n\n# ==============================================================================\n# 3. THỰC THI RE-RANK\n# ==============================================================================\nprint(\"⚡ Running Re-ranking Logic...\")\n\n# PENALTY_FACTOR: Hệ số phạt\n# 0.1: Phạt nhẹ (Gần như giữ nguyên)\n# 1.0 - 2.0: Phạt mạnh (Đẩy các bản sao xuống dứt khoát)\n# Khuyên dùng: 0.5 đến 1.5\ntop_ranked = re_rank(pl_test_features, pl_submission, penalty_factor=0.2)\n\n# ==============================================================================\n# 4. TẠO FINAL SUBMISSION\n# ==============================================================================\nprint(\"📝 Exporting Reranked Submission...\")\n\nfinal_sub = (\n    pl_submission.join(top_ranked, on=[\"Id\", \"ranker_id\"], how=\"left\")\n    .with_columns([\n        pl.col(\"new_selected\").alias(\"selected\") # Ghi đè rank cũ bằng rank mới\n    ])\n    .select([\"Id\", \"ranker_id\", \"selected\"])\n    .sort([\"ranker_id\", \"selected\"])\n)\n\n# Lưu file\nfinal_sub.write_csv(\"submission_reranked_postprocess.csv\")\nprint(\"✅ DONE! File: submission_reranked_postprocess.csv\")\n\n# 5. KIỂM TRA THAY ĐỔI (SỬA LỖI)\n# ==============================================================================\nprint(\"📊 Calculating Stats...\")\n\n# 1. Join lại bảng điểm gốc (pl_submission) với bảng rank mới (final_sub) theo Id\n# Để đảm bảo so sánh đúng dòng\ncompare_df = final_sub.join(pl_submission, on=[\"Id\", \"ranker_id\"], how=\"left\")\n\n# 2. Tính lại Rank gốc từ điểm số cũ (trong ngữ cảnh DataFrame)\ncompare_df = compare_df.with_columns(\n    pl.col(\"pred_score\")\n    .rank(method=\"ordinal\", descending=True)\n    .over(\"ranker_id\")\n    .cast(pl.Int32)\n    .alias(\"original_rank\")\n)\n\n# 3. Đếm số dòng khác biệt\ndiff_count = compare_df.filter(\n    pl.col(\"selected\") != pl.col(\"original_rank\")\n).height\n\nprint(f\"📊 Re-ranking changed positions of {diff_count} rows.\")\nprint(\"-\" * 60)\nprint(\"✅ QUY TRÌNH HOÀN TẤT! Bạn có thể nộp file 'submission_reranked_postprocess.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:36:23.680951Z","iopub.status.idle":"2025-12-27T08:36:23.681176Z","shell.execute_reply.started":"2025-12-27T08:36:23.681065Z","shell.execute_reply":"2025-12-27T08:36:23.681080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# import numpy as np\n# import pandas as pd\n# import gc\n# import os\n# import matplotlib.pyplot as plt\n\n# print(\"🚀 GIAI ĐOẠN 5: SINGLE LIGHTGBM (NATIVE API - GPU OPTIMIZED)\")\n\n# # ==============================================================================\n# # 1. CẤU HÌNH GPU & RAM (DÙNG CHO NATIVE API)\n# # ==============================================================================\n# # Đây là cấu hình 20% Stats (80% Train) tối ưu nhất\n# PARAMS_FINAL = {\n#     \"objective\": \"lambdarank\",\n#     \"metric\": \"ndcg\",\n#     \"eval_at\": [3],\n    \n#     # --- GPU ---\n#     \"device\": \"gpu\",\n#     \"gpu_platform_id\": 0,\n#     \"gpu_device_id\": 0,\n    \n#     # --- MEMORY SAFETY ---\n#     \"max_bin\": 63,              # Giảm bin để nhẹ VRAM\n#     \"two_round\": True,          # Load dữ liệu 2 vòng (Chống OOM)\n    \n#     # --- MODEL CAPACITY (255 LEAVES) ---\n#     \"num_leaves\": 255,          # Đủ sâu cho 8 triệu dòng dữ liệu\n#     \"min_data_in_leaf\": 300,    # Chống nhiễu\n#     \"learning_rate\": 0.03,\n#     \"n_estimators\": 3000,\n    \n#     # --- REGULARIZATION ---\n#     \"colsample_bytree\": 0.8,\n#     \"subsample\": 0.7,\n#     \"subsample_freq\": 1,\n#     \"reg_lambda\": 5.0,\n    \n#     \"random_state\": 42,\n#     \"n_jobs\": -1,\n#     \"verbose\": -1\n# }\n\n# # ==============================================================================\n# # 2. CHUẨN BỊ FEATURES\n# # ==============================================================================\n# DROP_LIST = [\n#     \"ranker_id\", \"selected\", \"fold\", \"group_size\", \"size_tier\", \"Id\", \n#     \"requestDate\", \"flight_hash\", \"searchRoute\",\n#     \"legs0_arrivalAt\", \"legs0_departureAt\", \"legs1_arrivalAt\", \"legs1_departureAt\"\n# ]\n\n# # Lọc feature\n# features = [c for c in pandas_df.columns if c not in DROP_LIST]\n\n# # Lọc feature hợp lệ (Số hoặc Category)\n# valid_features = []\n# for col in features:\n#     dtype = pandas_df[col].dtype\n#     if pd.api.types.is_numeric_dtype(dtype) or isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_bool_dtype(dtype):\n#         valid_features.append(col)\n# features = valid_features\n\n# # Xác định Category\n# cat_candidates = [\"pricingInfo_isAccessTP\", \"frequentFlyer\", \"isVip\", \"bySelf\", \"sex\", \n#                   \"cancellation_status\", \"exchange_status\", \"has_corporate_tariff\", \n#                   \"has_access_tp\", \"is_vip_freq\", \"is_one_way\", \"has_fees\", \n#                   \"is_major_carrier\", \"is_direct_leg1\"]\n# categorical_features = [c for c in cat_candidates if c in features]\n\n# print(f\"📋 Final Features: {len(features)}\")\n\n# # ==============================================================================\n# # 3. METRIC HITRATE (CHO NATIVE API)\n# # ==============================================================================\n# # Native API cần hàm metric dạng closure để đưa vào feval\n# import polars as pl\n# def get_hitrate_eval(val_ids):\n#     def eval_func(preds, eval_data):\n#         y_true = eval_data.get_label()\n#         # Tính toán bằng Polars cho nhanh\n#         df_tmp = pl.DataFrame({\"y\": y_true, \"p\": preds, \"g\": val_ids})\n#         hits = df_tmp.sort(\"p\", descending=True).group_by(\"g\").head(3).filter(pl.col(\"y\") == 1)\n#         valid_g = df_tmp.group_by(\"g\").len().filter(pl.col(\"len\") > 0)\n#         score = 0.0\n#         if valid_g.height > 0:\n#             score = hits.join(valid_g, on=\"g\", how=\"inner\").height / valid_g.height\n#         return 'hitrate', score, True\n#     return eval_func\n\n# # ==============================================================================\n# # 4. TRAINING LOOP (NATIVE API)\n# # ==============================================================================\n# final_test_preds = np.zeros(len(test_df_pd))\n# N_SPLITS = 10\n# all_history = [] # Lưu lịch sử để vẽ biểu đồ\n\n# print(f\"\\n🥊 Bắt đầu Train {N_SPLITS} Folds (GPU Mode)...\")\n\n# for fold in range(N_SPLITS):\n#     print(f\"\\n--- Fold {fold} ---\")\n    \n#     # Chỉ lấy index (Không copy dữ liệu)\n#     train_idx = pandas_df.index[pandas_df[\"fold\"] != fold]\n#     val_idx = pandas_df.index[pandas_df[\"fold\"] == fold]\n    \n#     # Lấy ID cho metric\n#     val_ranker_ids = pandas_df.loc[val_idx, \"ranker_id\"]\n#     if isinstance(val_ranker_ids.dtype, pd.CategoricalDtype):\n#         val_ranker_ids = val_ranker_ids.cat.codes.values\n#     else:\n#         val_ranker_ids = val_ranker_ids.values\n    \n#     # 1. Dataset Valid (Tạo trước cho nhẹ)\n#     dval = lgb.Dataset(\n#         pandas_df.loc[val_idx, features],\n#         label=pandas_df.loc[val_idx, \"selected\"],\n#         group=pandas_df.loc[val_idx].groupby(\"ranker_id\", sort=False).size().to_numpy(),\n#         categorical_feature=categorical_features,\n#         free_raw_data=True\n#     )\n    \n#     # 2. Dataset Train\n#     train_group = pandas_df.loc[train_idx].groupby(\"ranker_id\", sort=False).size().to_numpy()\n#     dtrain = lgb.Dataset(\n#         pandas_df.loc[train_idx, features],\n#         label=pandas_df.loc[train_idx, \"selected\"],\n#         group=train_group,\n#         categorical_feature=categorical_features,\n#         free_raw_data=True\n#     )\n    \n#     gc.collect()\n    \n#     # 3. Train\n#     evals_result = {} # Hứng lịch sử train\n    \n#     model = lgb.train(\n#         PARAMS_FINAL,\n#         dtrain,\n#         num_boost_round=3000,\n#         valid_sets=[dval],\n#         valid_names=['valid'],\n#          # <--- QUAN TRỌNG: Lưu history\n#         # feval=get_hitrate_eval(val_ranker_ids), # Bật dòng này nếu muốn xem hitrate (sẽ chậm hơn xíu)\n#         callbacks=[\n#             lgb.early_stopping(100, verbose=False),\n#             lgb.log_evaluation(500),\n#             lgb.record_evaluation(evals_result)\n#         ]\n#     )\n    \n#     # Lưu history\n#     all_history.append(evals_result['valid']['ndcg@3'])\n    \n#     # Log Score\n#     best_score = model.best_score['valid']['ndcg@3']\n#     print(f\"   🌟 Best NDCG@3: {best_score:.5f}\")\n    \n#     # Predict (Batch)\n#     final_test_preds += model.predict(test_df_pd[features]) / N_SPLITS\n    \n#     # Dọn dẹp\n#     del dtrain, dval, model, train_group\n#     gc.collect()\n\n# # ==============================================================================\n# # 5. VẼ BIỂU ĐỒ (CHO BÁO CÁO)\n# # ==============================================================================\n# print(\"\\n📊 Vẽ biểu đồ Training History...\")\n# plt.figure(figsize=(10, 6))\n\n# # Cắt về độ dài chung\n# min_len = min([len(h) for h in all_history])\n# trimmed = [h[:min_len] for h in all_history]\n# avg_hist = np.mean(trimmed, axis=0)\n\n# plt.plot(range(1, min_len+1), avg_hist, 'r-', linewidth=2, label='Average NDCG@3')\n# for h in all_history:\n#     plt.plot(range(1, len(h)+1), h, 'gray', alpha=0.3)\n\n# plt.title(f'LightGBM Training History (GPU - 20% Stats)\\nMean Best Score: {np.max(avg_hist):.5f}')\n# plt.xlabel('Iterations')\n# plt.ylabel('NDCG@3')\n# plt.legend()\n# plt.grid(True, alpha=0.3)\n# plt.savefig('training_chart.png')\n# plt.show()\n\n# # ==============================================================================\n# # 6. XUẤT FILE\n# # ==============================================================================\n# print(\"\\n📝 Exporting Submission...\")\n# test_df_pd[\"score\"] = final_test_preds\n# test_df_pd[\"rank\"] = test_df_pd.groupby(\"ranker_id\")[\"score\"].rank(method=\"first\", ascending=False).astype(int)\n\n# submission = test_df_pd[[\"Id\", \"ranker_id\", \"rank\"]].copy()\n# submission.rename(columns={\"rank\": \"selected\"}, inplace=True)\n\n# data_dir = \"../input/aeroclub-recsys-2025\"\n# if os.path.exists(f\"{data_dir}/test.parquet\"):\n#     original = pd.read_parquet(f\"{data_dir}/test.parquet\", columns=[\"Id\"])\n#     original['Id'] = original['Id'].astype(str)\n#     submission['Id'] = submission['Id'].astype(str)\n#     final_sub = original.merge(submission, on=\"Id\", how=\"left\")\n#     final_sub[\"selected\"] = final_sub[\"selected\"].fillna(1).astype(int)\n#     final_sub.to_csv(\"submission_single_gpu_20stats.csv\", index=False)\n# else:\n#     submission.to_csv(\"submission_single_gpu_20stats.csv\", index=False)\n\n# print(\"✅ DONE ALL!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T08:36:23.676312Z","iopub.status.idle":"2025-12-27T08:36:23.676605Z","shell.execute_reply.started":"2025-12-27T08:36:23.676455Z","shell.execute_reply":"2025-12-27T08:36:23.676467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# import numpy as np\n# import pandas as pd\n# import polars as pl\n# import gc\n\n# print(\"=\" * 60)\n# print(\"🚀 GIAI ĐOẠN TRAIN: LOW RAM + CUSTOM HITRATE METRIC\")\n# print(\"=\" * 60)\n\n# target = \"selected\"\n\n# # 1. Danh sách các cột hệ thống chắc chắn bỏ\n# DROP_LIST = [\n#     # ❌ NHÓM THỜI GIAN THÔ (THỦ PHẠM CHÍNH)\n#     \"legs0_arrivalAt\", \"legs0_departureAt\", \n#     \"legs1_arrivalAt\", \"legs1_departureAt\",\n#     \"requestDate\", \n#     \"legs0_segments0_departureFrom_airport_iata\", # Sân bay cụ thể quá cũng có thể gây nhiễu\n#     \"legs0_segments0_arrivalTo_airport_iata\",\n    \n#     # ❌ CÁC CỘT HỆ THỐNG\n#     \"ranker_id\", \"selected\", \"fold\", \"group_size\", \"size_tier\", \"Id\"\n# ]\n\n# # 2. Lấy tất cả cột, trừ cột hệ thống\n# initial_features = [c for c in pandas_df.columns if c not in DROP_LIST]\n\n# # 3. [FIX] Lọc bỏ các cột có kiểu dữ liệu không hợp lệ (Object, Datetime)\n# # LightGBM chỉ chấp nhận: number (int/float), bool, category\n# valid_features = []\n# print(\"⚙️ Checking feature dtypes...\")\n\n# for col in initial_features:\n#     dtype = pandas_df[col].dtype\n#     # Giữ lại nếu là số, category hoặc bool\n#     if pd.api.types.is_numeric_dtype(dtype) or isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_bool_dtype(dtype):\n#         valid_features.append(col)\n#     else:\n#         print(f\"   ❌ Dropping raw column: {col} ({dtype})\")\n\n# features = valid_features\n# print(f\"📋 Final Valid Features: {len(features)}\")\n# print(features)\n\n# # Xác định lại Categorical Features từ danh sách features sạch\n# cat_cols = [\"pricingInfo_isAccessTP\", \"frequentFlyer\", \"isVip\", \"bySelf\", \"sex\", \"cancellation_status\", \"exchange_status\", \"has_corporate_tariff\", \"has_access_tp\", \"is_vip_freq\", \"is_one_way\", \"has_fees\", \"is_major_carrier\", \"is_direct_leg1\"]\n# categorical_features = [c for c in cat_cols if c in features]\n\n# # 1. HÀM METRIC (CỦA BẠN - ĐÃ TỐI ƯU CHO NATIVE API)\n# # ==============================================================================\n# def hitrate_metric_polars(y_true, y_pred, group_ids):\n#     \"\"\" Hàm tính toán cốt lõi bằng Polars \"\"\"\n#     try:\n#         # Chuyển group_ids sang numpy array nếu chưa phải\n#         if hasattr(group_ids, 'values'): \n#             group_ids = group_ids.values\n            \n#         # Tạo Polars DataFrame (Zero-copy nếu có thể)\n#         df_tmp = pl.DataFrame({\n#             \"y\": y_true, \n#             \"p\": y_pred, \n#             \"g\": group_ids\n#         })\n        \n#         # Logic tính HitRate@3\n#         # 1. Sort theo điểm dự đoán giảm dần\n#         # 2. Lấy Top 3 mỗi nhóm\n#         # 3. Kiểm tra xem có vé y=1 trong đó không\n#         hits = df_tmp.sort(\"p\", descending=True).group_by(\"g\").head(3).filter(pl.col(\"y\") == 1)\n        \n#         # Đếm số nhóm hợp lệ (có nhiều hơn 0 vé)\n#         # Lưu ý: Logic cũ là >10, mình sửa thành >0 để bao quát hết, hoặc giữ >10 tùy bạn\n#         valid_g = df_tmp.group_by(\"g\").len().filter(pl.col(\"len\") > 0)\n        \n#         if valid_g.height == 0: \n#             return 0.0\n            \n#         return hits.join(valid_g, on=\"g\", how=\"inner\").height / valid_g.height\n#     except Exception as e:\n#         print(f\"Metric Error: {e}\")\n#         return 0.0\n\n# # ==============================================================================\n# # 2. TRAINING LOOP\n# # ==============================================================================\n# final_test_preds = np.zeros(len(test_df_pd))\n# N_SPLITS = 10\n\n# # Cấu hình Native (Đã tối ưu RAM)\n# PARAMS_LOW_RAM = {\n#     \"objective\": \"lambdarank\",\n#     \"metric\": \"ndcg\",\n#     \"eval_at\": [3],\n#     \"max_bin\": 63,\n#     \"n_estimators\": 3000,   \n#     \"learning_rate\": 0.03,\n#     \"num_leaves\": 127,\n#     \"min_data_in_leaf\": 101, \n#     \"reg_lambda\": 2.0,\n#     \"colsample_bytree\": 0.8,\n#     \"subsample\": 0.7,\n#     \"subsample_freq\": 1,\n#     'lambdarank_truncation_level': 10,\n#     \"random_state\": 42,\n#     \"n_jobs\": -1,\n#     \"two_round\": True,\n#     \"verbose\": -1\n# }\n\n# print(f\"\\n🥊 Bắt đầu Train {N_SPLITS} Folds...\")\n\n# for fold in range(N_SPLITS):\n#     print(f\"\\n--- Fold {fold} ---\")\n    \n#     train_mask = pandas_df[\"fold\"] != fold\n#     val_mask = pandas_df[\"fold\"] == fold\n    \n#     # --- CHUẨN BỊ ID CHO METRIC ---\n#     # Native API không tự lưu trữ ranker_id, ta phải truyền nó vào hàm metric\n#     # Lấy ID của tập Valid và ép kiểu an toàn\n#     val_ranker_ids = pandas_df.loc[val_mask, \"ranker_id\"]\n#     if isinstance(val_ranker_ids.dtype, pd.CategoricalDtype):\n#         val_ranker_ids = val_ranker_ids.cat.codes.values # Dùng mã số (int) cho nhanh\n#     else:\n#         val_ranker_ids = val_ranker_ids.values\n\n#     # --- WRAPPER FUNCTION (CẦU NỐI) ---\n#     # Hàm này kết nối LightGBM Native với hàm Polars của bạn\n#     def lgb_hitrate_eval(preds, eval_data):\n#         # preds: Mảng điểm dự đoán từ model\n#         # eval_data: Dataset chứa label thực tế\n#         y_true = eval_data.get_label()\n        \n#         # Gọi hàm tính toán Polars\n#         # Lưu ý: val_ranker_ids được lấy từ scope bên ngoài (closure)\n#         score = hitrate_metric_polars(y_true, preds, val_ranker_ids)\n        \n#         # Trả về format chuẩn: (tên, giá trị, càng_cao_càng_tốt)\n#         return 'hitrate', score, True\n\n#     # --- TẠO DATASET ---\n#     # Valid Set\n#     dval = lgb.Dataset(\n#         pandas_df.loc[val_mask, features], \n#         label=pandas_df.loc[val_mask, \"selected\"],\n#         group=pandas_df.loc[val_mask].groupby(\"ranker_id\", sort=False).size().to_numpy(),\n#         categorical_feature=categorical_features,\n#         free_raw_data=True\n#     )\n    \n#     # Train Set\n#     train_group = pandas_df.loc[train_mask].groupby(\"ranker_id\", sort=False).size().to_numpy()\n#     dtrain = lgb.Dataset(\n#         pandas_df.loc[train_mask, features], \n#         label=pandas_df.loc[train_mask, \"selected\"],\n#         group=train_group,\n#         categorical_feature=categorical_features,\n#         free_raw_data=True \n#     )\n    \n#     gc.collect()\n    \n#     # --- TRAIN VỚI CUSTOM METRIC ---\n#     model = lgb.train(\n#         PARAMS_LOW_RAM,\n#         dtrain,\n#         num_boost_round=2000,\n#         valid_sets=[dval],\n#         valid_names=['valid'],\n#         # 🔥 Đưa hàm metric vào đây\n#         feval=lgb_hitrate_eval, \n#         callbacks=[\n#             lgb.early_stopping(50, verbose=False),\n#             lgb.log_evaluation(0) # Tắt log chi tiết\n#         ]\n#     )\n    \n#     # Log Kết quả\n#     # Lấy điểm tốt nhất (lưu ý key là 'hitrate')\n#     best_score = model.best_score['valid']['hitrate']\n#     print(f\"   🌟 Best HitRate Fold {fold}: {best_score:.4f}\")\n    \n#     # Predict\n#     final_test_preds += model.predict(test_df_pd[features]) / N_SPLITS\n    \n#     # Dọn dẹp\n#     del dtrain, dval, model, train_group, val_ranker_ids\n#     gc.collect()\n\n# # ==============================================================================\n# # 3. XUẤT FILE (GIỮ NGUYÊN)\n# # ==============================================================================\n# # ... (Phần xuất file như cũ)\n# print(\"\\n📝 Exporting...\")\n# test_df_pd[\"score\"] = final_test_preds\n# test_df_pd[\"rank\"] = test_df_pd.groupby(\"ranker_id\")[\"score\"].rank(method=\"first\", ascending=False).astype(int)\n\n# submission = test_df_pd[[\"Id\", \"ranker_id\", \"rank\"]].copy()\n# submission.rename(columns={\"rank\": \"selected\"}, inplace=True)\n\n# data_dir = \"../input/aeroclub-recsys-2025\"\n# if os.path.exists(f\"{data_dir}/test.parquet\"):\n#     original = pd.read_parquet(f\"{data_dir}/test.parquet\", columns=[\"Id\"])\n#     original['Id'] = original['Id'].astype(str)\n#     submission['Id'] = submission['Id'].astype(str)\n    \n#     final_sub = original.merge(submission, on=\"Id\", how=\"left\")\n#     final_sub[\"selected\"] = final_sub[\"selected\"].fillna(1).astype(int)\n#     final_sub.to_csv(\"submission_lowram_hitrate.csv\", index=False)\n# else:\n#     submission.to_csv(\"submission_lowram_hitrate.csv\", index=False)\n\n# print(\"✅ DONE! File: submission_lowram_hitrate.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Selection","metadata":{"_uuid":"9b3d8e25-106c-4039-8490-c55a844cf3ee","_cell_guid":"1058f4d0-cbcc-4965-b824-5113b77d2041","trusted":true,"collapsed":false,"id":"NmcPTUOvDxtO","jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"","metadata":{"_uuid":"5326090c-7a78-4ade-bb4b-7ab2dfedcd9b","_cell_guid":"399ed7cc-11de-4f73-90d9-5edd2ee5319a","trusted":true,"collapsed":false,"id":"bPDoRFGBn8Z6","jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}