{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4436180,"sourceType":"datasetVersion","datasetId":2597726},{"sourceId":4483558,"sourceType":"datasetVersion","datasetId":2623568},{"sourceId":13274305,"sourceType":"datasetVersion","datasetId":8384420},{"sourceId":13819184,"sourceType":"datasetVersion","datasetId":8770990},{"sourceId":13929603,"sourceType":"datasetVersion","datasetId":8876655}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Otto RecSys - Candidate ReRank Model - Use source code","metadata":{"_uuid":"6b39da49-a2df-462b-96fd-ab41ad74959d","_cell_guid":"1b756def-d852-4994-aed8-25f06c7fcef5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Data Preparing","metadata":{"_uuid":"e8cfb350-2c98-43ea-add0-0f94a9d1a134","_cell_guid":"a22bf930-8cf4-4893-8d2f-0d7e50cf34aa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!git clone https://github.com/triet4p/otto-rec-sys.git","metadata":{"_uuid":"f1e7225e-db97-40e2-b6da-95dda9f5ca17","_cell_guid":"52dd34df-ac5a-434a-bc56-7f47c0ad2515","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:06:25.996106Z","iopub.execute_input":"2025-12-28T14:06:25.996471Z","iopub.status.idle":"2025-12-28T14:06:27.046385Z","shell.execute_reply.started":"2025-12-28T14:06:25.996442Z","shell.execute_reply":"2025-12-28T14:06:27.045211Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd otto-rec-sys\n!git checkout 33986ff48aa765cb612d02b94565c7818c0fbfe6","metadata":{"_uuid":"b7fce74c-2d8e-45b9-adc9-ce96f1d60ddc","_cell_guid":"bbe9a6dd-5931-42e9-934c-9145296c05f2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:06:27.048388Z","iopub.execute_input":"2025-12-28T14:06:27.048823Z","iopub.status.idle":"2025-12-28T14:06:27.195033Z","shell.execute_reply.started":"2025-12-28T14:06:27.048776Z","shell.execute_reply":"2025-12-28T14:06:27.193716Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/working/otto-rec-sys')","metadata":{"_uuid":"b5d716a3-d850-41e8-a0b9-f924dc20196a","_cell_guid":"c28e1ed1-9259-4c29-8d5c-748969169dd1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:06:27.196415Z","iopub.execute_input":"2025-12-28T14:06:27.196813Z","iopub.status.idle":"2025-12-28T14:06:27.202147Z","shell.execute_reply.started":"2025-12-28T14:06:27.196772Z","shell.execute_reply":"2025-12-28T14:06:27.201250Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nfrom src.core.utils import load_raw_data_parquet\n# Load train parquet\n\ntrain_df = load_raw_data_parquet('/kaggle/input/otto-chunk-data-inparquet-format/train_parquet/*')","metadata":{"_uuid":"da3bc7a4-6ddd-4c83-a484-5e1de2485a1f","_cell_guid":"b0b067ab-8b6b-4ba1-a0ca-24d5c2f33d92","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:06:27.204271Z","iopub.execute_input":"2025-12-28T14:06:27.204538Z","iopub.status.idle":"2025-12-28T14:06:55.113096Z","shell.execute_reply.started":"2025-12-28T14:06:27.204516Z","shell.execute_reply":"2025-12-28T14:06:55.112057Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nfrom src.candidate_generation.loader import load_covisit_matrix\nMATRIX_BASE_PATH = '/kaggle/input/otto-precal-covisit-matrices/submission/train/'\n\ndf_clicks_train = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_clicks_*.pqt', 'clicks', 20)\ndf_buys_train = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_carts_orders_*.pqt', 'buys', 15)\ndf_buy2buy_train = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_buy2buy_*.pqt', 'buy2buy', 15)","metadata":{"_uuid":"409222fc-e99f-4976-af41-6ed0287648f2","_cell_guid":"c6ff3cfc-4bf2-4311-ba52-91763c7cff68","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:06:55.114129Z","iopub.execute_input":"2025-12-28T14:06:55.114410Z","iopub.status.idle":"2025-12-28T14:07:23.144138Z","shell.execute_reply.started":"2025-12-28T14:06:55.114386Z","shell.execute_reply":"2025-12-28T14:07:23.143232Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from src.pipeline.preprocess import sample_train\nTRAIN_SESSION_SAMPLE_RATE = 0.18 # Lấy 40% số session\n\nsampled_train_dfs = []\nseeds = [42,43,44]\n\nfor i in range(len(seeds)):\n    sampled_train_dfs.append(sample_train(train_df,\n                                          TRAIN_SESSION_SAMPLE_RATE,\n                                          seed=seeds[i]))","metadata":{"_uuid":"0d00e652-278d-4ba3-ab04-5b7edb3d9c57","_cell_guid":"e87ad537-3fc1-4bbc-8f7d-c58b5d9a6674","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:07:23.145188Z","iopub.execute_input":"2025-12-28T14:07:23.145529Z","iopub.status.idle":"2025-12-28T14:08:06.874680Z","shell.execute_reply.started":"2025-12-28T14:07:23.145499Z","shell.execute_reply":"2025-12-28T14:08:06.873803Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nfrom src.pipeline.preprocess import get_history_and_label_df\nhistory_dfs = []\nhistory_source_dfs = []\ntruth_label_dfs = []\n\nfor i in range(len(seeds)):\n    history_df, truth_label_df, history_source_df = get_history_and_label_df(sampled_train_dfs[i])\n    history_dfs.append(history_df)\n    truth_label_dfs.append(truth_label_df)\n    history_source_dfs.append(history_source_df)\n\n    del history_df, truth_label_df\n    gc.collect()","metadata":{"_uuid":"f3bb88c9-6b72-4ff6-a6cd-5d8e80fe7062","_cell_guid":"8e90eb3b-7c86-4995-a785-e279ed9a739a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:08:06.875889Z","iopub.execute_input":"2025-12-28T14:08:06.876670Z","iopub.status.idle":"2025-12-28T14:08:25.422054Z","shell.execute_reply.started":"2025-12-28T14:08:06.876634Z","shell.execute_reply":"2025-12-28T14:08:25.421046Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from src.pipeline.preprocess import get_popular_items_df\n\npopular_items_dfs = []\nfor i in range(len(seeds)):\n    popular_items_dfs.append(get_popular_items_df(sampled_train_dfs[i]))","metadata":{"_uuid":"ec695fbf-209a-4120-a956-e22a5290b74c","_cell_guid":"4728cf0c-b9d8-4800-9628-6c79343022bb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:08:25.423019Z","iopub.execute_input":"2025-12-28T14:08:25.423331Z","iopub.status.idle":"2025-12-28T14:08:35.713782Z","shell.execute_reply.started":"2025-12-28T14:08:25.423300Z","shell.execute_reply":"2025-12-28T14:08:35.713010Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from src.pipeline.preprocess import pre_compute_item_popularity\nitem_popularity_dfs = []\nfor i in range(len(seeds)):\n    item_popularity_dfs.append(pre_compute_item_popularity(sampled_train_dfs[i]))","metadata":{"_uuid":"caad5d34-90cb-44c8-9eb1-efb069030657","_cell_guid":"0fc96977-f1ab-42ed-acfd-ed790ac0fe60","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:08:35.714911Z","iopub.execute_input":"2025-12-28T14:08:35.715244Z","iopub.status.idle":"2025-12-28T14:09:00.725520Z","shell.execute_reply.started":"2025-12-28T14:08:35.715218Z","shell.execute_reply":"2025-12-28T14:09:00.724502Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport polars as pl\nfrom tqdm import tqdm\nfrom src.pipeline.preprocess import process_chunk\n\nN_CHUNKS = 20\nfor i in range(len(seeds)):\n    history_df = history_dfs[i]\n    popular_items_df = popular_items_dfs[i]\n    all_sessions = history_df['session'].unique().to_list()\n    chunk_size = len(all_sessions) // N_CHUNKS\n\n    TEMP_CHUNK_PATH = f'/kaggle/working/temp_candidate_chunks/sample_{i}/' # Thư mục để lưu các file tạm\n    \n    os.makedirs(TEMP_CHUNK_PATH, exist_ok=True)\n    \n    print(f\"\\n--- Processing {len(all_sessions)} sessions in {N_CHUNKS+1} chunks ---\")\n    for i in tqdm(range(N_CHUNKS + 1)):\n        start = i * chunk_size\n        end = (i + 1) * chunk_size\n        if start >= len(all_sessions):\n            break\n        \n        session_chunk_ids = all_sessions[start:end]\n        history_chunk = history_df.filter(pl.col('session').is_in(session_chunk_ids))\n        \n        # Gọi hàm xử lý cho chunk\n        chunk_result = process_chunk(history_chunk, popular_items_df,\n                                     df_clicks_train, df_buys_train, df_buy2buy_train)\n        # Thêm đặc trưng cuối cùng cho nguồn popular\n        chunk_result = chunk_result.with_columns(\n            pl.col('candidate_aid').is_in(popular_items_df['candidate_aid']).cast(pl.UInt8).alias('source_popular')\n        )\n        # --- THAY ĐỔI QUAN TRỌNG: LƯU RA FILE THAY VÌ APPEND VÀO LIST ---\n        chunk_result.write_parquet(TEMP_CHUNK_PATH + f'candidates_chunk_{i}.pqt')\n        \n        # Dọn dẹp bộ nhớ\n        gc.collect()","metadata":{"_uuid":"dbbe1d94-19fd-4bc8-bd62-98130dfb8e2a","_cell_guid":"2459a3de-23f5-468c-9289-fa9416c414e3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:09:00.729662Z","iopub.execute_input":"2025-12-28T14:09:00.730251Z","iopub.status.idle":"2025-12-28T14:21:13.250697Z","shell.execute_reply.started":"2025-12-28T14:09:00.730222Z","shell.execute_reply":"2025-12-28T14:21:13.249835Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del df_clicks_train, df_buys_train, df_buy2buy_train, popular_items_dfs, sampled_train_dfs\n_ = gc.collect()","metadata":{"_uuid":"46147a49-36c7-4305-b1d9-048ca53e83d7","_cell_guid":"7da745a6-f0a9-4cd7-b3b7-55139b420df8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:21:13.251729Z","iopub.execute_input":"2025-12-28T14:21:13.252006Z","iopub.status.idle":"2025-12-28T14:21:13.623893Z","shell.execute_reply.started":"2025-12-28T14:21:13.251975Z","shell.execute_reply":"2025-12-28T14:21:13.622804Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nnp.random.seed(42)","metadata":{"_uuid":"f5bc6bd4-9833-4d92-85b4-111fdcf4fb97","_cell_guid":"19f5b455-43ef-4a2e-85ff-484e36c63b0c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:21:13.624963Z","iopub.execute_input":"2025-12-28T14:21:13.625226Z","iopub.status.idle":"2025-12-28T14:21:13.642825Z","shell.execute_reply.started":"2025-12-28T14:21:13.625204Z","shell.execute_reply":"2025-12-28T14:21:13.641689Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training","metadata":{"_uuid":"47157c3f-cf13-4f2b-a6bb-78b9088afde5","_cell_guid":"5afc297c-1d29-4eb3-82c5-bcc78d256c2f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# 4. Tạo ra 3 bộ dữ liệu huấn luyện riêng biệt\n# Định nghĩa các siêu tham số cho việc lấy mẫu\nTARGET_RATIOS = {'clicks': 10, 'carts': 20, 'orders': 30}\nPOS_RATE = {'clicks': 0.5, 'carts': 1.0, 'orders': 1.0}\nPOPULAR_FRACTION = 0.5 # 50% mẫu âm sẽ là \"khó\", 50% là ngẫu nhiên","metadata":{"_uuid":"5af2d0fc-1f04-4576-9a11-bb2d97130c4e","_cell_guid":"b742e41c-8e63-4adb-a6de-4a7d043c7f06","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:21:13.643859Z","iopub.execute_input":"2025-12-28T14:21:13.644123Z","iopub.status.idle":"2025-12-28T14:21:13.662086Z","shell.execute_reply.started":"2025-12-28T14:21:13.644102Z","shell.execute_reply":"2025-12-28T14:21:13.661177Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nfrom src.pipeline.training import create_training_set_for_type\nfor i in range(len(seeds)):\n    TEMP_CHUNK_PATH = f'/kaggle/working/temp_candidate_chunks/sample_{i}/'\n    lazy_final_df = pl.scan_parquet(TEMP_CHUNK_PATH + 'candidates_chunk_*.pqt')\n    lazy_final_df = lazy_final_df.fill_null(0)\n\n    for pred_type in ['clicks', 'carts', 'orders']:\n        training_sets = create_training_set_for_type(\n            lazy_final_df, truth_label_dfs[i], item_popularity_dfs[i], pred_type,\n            positive_rate = POS_RATE[pred_type],\n            target_neg_pos_ratio=TARGET_RATIOS[pred_type], popular_fraction=POPULAR_FRACTION\n        )\n        training_sets.write_parquet(TEMP_CHUNK_PATH + f'training_set_{pred_type}.pqt')\n        del training_sets\n        gc.collect()","metadata":{"_uuid":"ecdb6be3-6f0b-4475-9feb-51626a6a49e4","_cell_guid":"e9fed50f-7a45-4816-9b9e-5af63d19d931","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:21:13.663281Z","iopub.execute_input":"2025-12-28T14:21:13.663592Z","iopub.status.idle":"2025-12-28T14:31:55.994755Z","shell.execute_reply.started":"2025-12-28T14:21:13.663563Z","shell.execute_reply":"2025-12-28T14:31:55.992638Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport lightgbm as lgb\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# --- 1. Feature Tĩnh (Item Features) ---\ndef create_time_window_features(feature_source_df: pl.DataFrame, time_window_days: int = None) -> pl.DataFrame:\n    if time_window_days is not None:\n        last_ts = feature_source_df['ts'].max()\n        start_ts = last_ts - (time_window_days * 24 * 60 * 60)\n        source_df = feature_source_df.filter(pl.col('ts') >= start_ts)\n        suffix = f'_{time_window_days}d'\n    else:\n        source_df = feature_source_df\n        suffix = '_all'\n        \n    item_feats = source_df.group_by('aid').agg([\n        pl.count().alias(f'item_total_counts{suffix}'),\n        pl.col('type').filter(pl.col('type') == 0).count().alias(f'item_click_counts{suffix}'),\n        pl.col('type').filter(pl.col('type') == 1).count().alias(f'item_cart_counts{suffix}'),\n        pl.col('type').filter(pl.col('type') == 2).count().alias(f'item_order_counts{suffix}'),\n    ]).rename({'aid': 'candidate_aid'})\n    \n    # Tính tỷ lệ chuyển đổi (Smoothing +10 để tránh nhiễu ở item ít tương tác)\n    item_feats = item_feats.with_columns([\n        (pl.col(f'item_order_counts{suffix}') / (pl.col(f'item_click_counts{suffix}') + 10)).alias(f'item_buy_ratio{suffix}'),\n        (pl.col(f'item_cart_counts{suffix}') / (pl.col(f'item_click_counts{suffix}') + 10)).alias(f'item_cart_ratio{suffix}'),\n    ])\n    return item_feats\n\n# --- 2. Feature Động (Session Context) ---\ndef create_session_context_features(session_context_df: pl.DataFrame) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:\n    \"\"\"\n    Tính toán tất cả các feature dựa trên context của session.\n    Trả về: (session_level_features, interaction_level_features, last_item_info)\n    \"\"\"\n    # A. Session Level Features\n    session_feats = session_context_df.group_by('session').agg([\n        pl.count().alias('session_length'),\n        pl.col('aid').n_unique().alias('session_unique_aids'),\n        pl.col('ts').max().alias('session_end_ts'),\n        (pl.col('ts').max() - pl.col('ts').min()).alias('session_duration'),\n    ])\n    \n    # B. Interaction Level Features (Lặp lại & Thời gian cuối)\n    interaction_feats = session_context_df.group_by(['session', 'aid']).agg([\n        pl.count().alias('num_repetitions'),\n        pl.col('ts').max().alias('last_item_ts')\n    ]).rename({'aid': 'candidate_aid'})\n    \n    # C. Last Item Info (Item cuối cùng user xem)\n    last_items = session_context_df.sort('ts').group_by('session', maintain_order=True).last()\n    last_items = last_items.select(['session', 'aid']).rename({'aid': 'last_aid'})\n    \n    return session_feats, interaction_feats, last_items","metadata":{"_uuid":"b0787b83-800d-463b-ace2-4ffe24cb9175","_cell_guid":"43fe84a5-d1a0-4a39-b603-9ec1a98ca041","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:31:56.033227Z","iopub.execute_input":"2025-12-28T14:31:56.033575Z","iopub.status.idle":"2025-12-28T14:31:56.455639Z","shell.execute_reply.started":"2025-12-28T14:31:56.033549Z","shell.execute_reply":"2025-12-28T14:31:56.454455Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_sorted_rank_features(df: pl.DataFrame, fill_value: int = 999) -> pl.DataFrame:\n    \"\"\"Tạo feature rank tốt nhất từ tất cả các nguồn.\"\"\"\n    rank_cols = [col for col in df.columns if col.startswith('rank_')]\n    if not rank_cols: return df\n\n    # Tạo list rank, sort và lấy ra các giá trị min\n    rank_exprs = [pl.col(c).fill_null(fill_value) for c in rank_cols]\n    \n    df = df.with_columns(\n        pl.concat_list(rank_exprs).list.sort().alias('temp_sorted_ranks')\n    )\n    \n    # Tách ra min_rank_1 (best), min_rank_2 (2nd best)\n    new_cols = [\n        pl.col('temp_sorted_ranks').list.get(0).alias('min_rank_1'),\n        pl.col('temp_sorted_ranks').list.get(1).alias('min_rank_2'),\n        # Số lượng nguồn gợi ý item này\n        (pl.col('temp_sorted_ranks').list.eval(pl.element() < fill_value).list.sum()).alias('n_sources_present')\n    ]\n    return df.with_columns(new_cols).drop('temp_sorted_ranks')\n\ndef add_features(df: pl.DataFrame, \n                 time_window_feats_list: list[pl.DataFrame], # [item_all, item_7d]\n                 session_feats: pl.DataFrame,\n                 interaction_feats: pl.DataFrame,\n                 last_items: pl.DataFrame) -> pl.DataFrame:\n   \n    # 1. Join các bảng gốc (Giữ nguyên)\n    for tw_feat in time_window_feats_list:\n        df = df.join(tw_feat, on='candidate_aid', how='left')\n    df = df.join(session_feats, on='session', how='left')\n    df = df.join(interaction_feats, on=['session', 'candidate_aid'], how='left')\n    df = df.join(last_items, on='session', how='left')\n\n    # --- NHÓM 1: TREND / VELOCITY (Tốc độ tăng trưởng của Item) ---\n    # So sánh 7 ngày vs All time. \n    # Logic: Item có tỷ trọng click trong 7 ngày cao bất thường so với lịch sử -> Đang Hot.\n    \n    # Giả sử time_window_feats_list[0] là ALL, [1] là 7D\n    # Các cột sẽ có suffix '_all' và '_7d'\n    \n    df = df.with_columns([\n        # Tỷ lệ click gần đây / click tổng (Cộng 10 để tránh chia 0 và nhiễu)\n        (pl.col('item_click_counts_7d').fill_null(0) / (pl.col('item_click_counts_all').fill_null(0) + 10)).alias('click_trend_7d_vs_all'),\n        \n        # Tỷ lệ order gần đây / order tổng\n        (pl.col('item_order_counts_7d').fill_null(0) / (pl.col('item_order_counts_all').fill_null(0) + 10)).alias('order_trend_7d_vs_all'),\n        \n        # Conversion Rate thay đổi thế nào? (CR 7 ngày - CR All)\n        (pl.col('item_buy_ratio_7d').fill_null(0) - pl.col('item_buy_ratio_all').fill_null(0)).alias('conversion_trend_diff')\n    ])\n\n    # --- NHÓM 2: CROSS-SOURCE COMPARISON (So sánh giữa các nguồn Co-visit) ---\n    # Logic: Sự chênh lệch thứ hạng giữa các nguồn nói lên điều gì?\n    # Ví dụ: Rank Buy2Buy thấp (tốt) nhưng Rank Clicks cao (tệ) -> Item này ít người click nhưng hễ click là mua -> Tiềm năng cao.\n    \n    # Fill null rank bằng 999 trước khi tính toán\n    rank_cols = ['rank_clicks', 'rank_buys', 'rank_buy2buy']\n    for c in rank_cols:\n        if c not in df.columns:\n            df = df.with_columns(pl.lit(999).alias(c))\n        else:\n            df = df.with_columns(pl.col(c).fill_null(999))\n\n    df = df.with_columns([\n        # Chênh lệch rank\n        (pl.col('rank_clicks') - pl.col('rank_buy2buy')).alias('rank_diff_click_buy2buy'),\n        (pl.col('rank_buys') - pl.col('rank_buy2buy')).alias('rank_diff_buys_buy2buy'),\n        \n        # Tổng hợp trọng số (Weighted Sum) - Tạo ra một \"Siêu điểm số\"\n        (pl.col('wgt_buy2buy').fill_null(0) * 2 + pl.col('wgt_buys').fill_null(0) * 1).alias('combined_buy_weight')\n    ])\n\n    # --- NHÓM 3: CONTEXTUAL RECENCY (Tính gần đây kết hợp ngữ cảnh) ---\n    # Logic: Recency quan trọng, nhưng Recency của một item \"Hot\" quan trọng hơn Recency của item \"Rác\".\n    \n    # Tính Recency cơ bản trước\n    df = df.with_columns(\n        (pl.col('session_end_ts') - pl.col('last_item_ts')).fill_null(7*24*3600).alias('recency_score')\n    )\n    \n    # Log Recency để giảm biên độ số (giây -> log giây)\n    df = df.with_columns(\n        pl.col('recency_score').log1p().alias('log_recency_score')\n    )\n    \n    # Tương tác: Điểm Co-visit chia cho thời gian (Càng gần càng giá trị)\n    # Thêm 1 vào log_recency để tránh chia 0\n    df = df.with_columns([\n        (pl.col('wgt_buy2buy').fill_null(0) / (pl.col('log_recency_score') + 1)).alias('wgt_buy2buy_decayed'),\n        (pl.col('wgt_clicks').fill_null(0) / (pl.col('log_recency_score') + 1)).alias('wgt_clicks_decayed')\n    ])\n\n    # --- NHÓM 4: CÁC CỜ (FLAGS) QUAN TRỌNG (Giữ lại từ cũ) ---\n    df = df.with_columns([\n        (pl.col('candidate_aid') == pl.col('last_aid')).cast(pl.Int8).fill_null(0).alias('is_last_viewed'),\n        # Item này có phải là item phổ biến nhất trong session không? (Logic đơn giản: count > 1)\n        (pl.col('num_repetitions') > 1).cast(pl.Int8).alias('is_repeated_in_session')\n    ])\n\n    # --- Sorted Ranks (Giữ nguyên - rất mạnh) ---\n    df = add_sorted_rank_features(df)\n\n    # Dọn dẹp\n    df = df.fill_null(0)\n    cols_to_drop = ['session_end_ts', 'last_item_ts', 'last_aid', 'session_duration', 'first_item_ts'] \n    df = df.drop([c for c in cols_to_drop if c in df.columns])\n    \n    return df","metadata":{"_uuid":"fc89d010-624a-4a31-98af-e1c37a5699c7","_cell_guid":"d19263ba-13ac-4116-9c5e-18f70e4aaad5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:31:56.457029Z","iopub.execute_input":"2025-12-28T14:31:56.457664Z","iopub.status.idle":"2025-12-28T14:31:56.474323Z","shell.execute_reply.started":"2025-12-28T14:31:56.457592Z","shell.execute_reply":"2025-12-28T14:31:56.473214Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def select_best_features(df: pl.DataFrame, target_type: str, top_k: int = 60):\n    \"\"\"\n    Huấn luyện nhanh 1 model để chọn ra top_k features tốt nhất cho target_type.\n    \"\"\"\n    print(f\"  >> Performing Feature Selection for {target_type}...\")\n    \n    ignore_cols = ['session', 'candidate_aid', 'label']\n    feature_cols = [c for c in df.columns if c not in ignore_cols]\n    \n    # Sample dữ liệu để chạy nhanh (ví dụ 2 triệu dòng)\n    if len(df) > 2_000_000:\n        df_sample = df.sample(n=2_000_000, seed=42)\n    else:\n        df_sample = df\n        \n    X = df_sample.select(feature_cols).to_numpy()\n    y = df_sample.select('label').to_numpy().ravel()\n    groups = df_sample.group_by('session', maintain_order=True).len()['len'].to_numpy()\n    \n    # Train model nhẹ\n    model = lgb.LGBMRanker(\n        objective=\"lambdarank\", metric=\"map\",\n        n_estimators=50, learning_rate=0.1, max_depth=5,\n        importance_type='gain', random_state=42, n_jobs=-1\n    )\n    model.fit(X, y, group=groups)\n    \n    # Lấy feature importance\n    imp_df = pl.DataFrame({\n        'feature': feature_cols,\n        'gain': model.feature_importances_\n    }).sort('gain', descending=True)\n    \n    # Chọn top K\n    best_feats = imp_df.head(top_k)['feature'].to_list()\n    print(f\"     Selected {len(best_feats)} features. Top 5: {best_feats[:5]}\")\n    \n    # (Tùy chọn) In ra các feature bị loại bỏ để kiểm tra\n    # dropped = [f for f in feature_cols if f not in best_feats]\n    # print(f\"     Dropped: {dropped[:5]}...\")\n    \n    return best_feats\n\ndef train_final_model(df: pl.DataFrame, features: list, model_type: str):\n    \"\"\"Huấn luyện model chính thức với danh sách feature đã chọn.\"\"\"\n    print(f\"  >> Training Final Model for {model_type} with {len(features)} features...\")\n    \n    df = df.sort('session')\n    X = df.select(features).to_numpy()\n    y = df.select('label').to_numpy().ravel()\n    groups = df.group_by('session', maintain_order=True).len()['len'].to_numpy()\n    \n    model = lgb.LGBMRanker(\n        objective=\"lambdarank\", metric=\"map\",\n        n_estimators=500, learning_rate=0.05, num_leaves=32,\n        subsample=0.8, colsample_bytree=0.7,\n        random_state=42, n_jobs=-1\n    )\n    \n    # --- SỬA LỖI Ở ĐÂY ---\n    # Thêm eval_set và eval_group\n    model.fit(\n        X, \n        y, \n        group=groups, \n        eval_set=[(X, y)],       # Đưa tập train vào làm tập đánh giá\n        eval_group=[groups],     # Cung cấp thông tin group cho tập đánh giá\n        callbacks=[lgb.early_stopping(50, verbose=False)] # Tăng patience lên 50 cho an toàn\n    )\n    return model","metadata":{"_uuid":"33e4eb2b-ad7b-4b4f-a115-de84e25d9ce8","_cell_guid":"b7d6379c-3226-4a3c-b3d1-1d584e3d2b95","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:31:56.475689Z","iopub.execute_input":"2025-12-28T14:31:56.475976Z","iopub.status.idle":"2025-12-28T14:31:56.503585Z","shell.execute_reply.started":"2025-12-28T14:31:56.475955Z","shell.execute_reply":"2025-12-28T14:31:56.502504Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# Dictionary để lưu danh sách feature tốt nhất cho mỗi loại (sẽ được điền ở vòng lặp 0)\nbest_features_map = {} \ntrained_models_lst = [{} for _ in range(len(seeds))] \n\n# Giả sử history_source_dfs đã có sẵn từ các bước trước\nfor i in range(len(seeds)):\n    print(f\"\\n{'='*30}\\nPROCESSING SAMPLE {i}\\n{'='*30}\")\n    \n    TEMP_CHUNK_PATH = f'/kaggle/working/temp_candidate_chunks/sample_{i}/'\n    \n    # 1. Chuẩn bị Features (Tĩnh & Động)\n    history_source_df = history_source_dfs[i]\n    \n    # Feature Tĩnh (Item) - Tính trên history_source_df (để tránh leak)\n    time_window_feats = [\n        create_time_window_features(history_source_df, None),\n        create_time_window_features(history_source_df, 7)\n    ]\n    \n    # Feature Động (Session) - Tính trên history_source_df\n    session_feats, interaction_feats, last_items = create_session_context_features(history_source_df)\n\n    # ===================================================================\n    # GIAI ĐOẠN 1: HUẤN LUYỆN CLICKS\n    # ===================================================================\n    print(\"\\n--- STAGE 1: Training CLICKS model ---\")\n    df_train_clicks = pl.read_parquet(TEMP_CHUNK_PATH + 'training_set_clicks.pqt')\n    df_train_clicks = add_features(df_train_clicks, time_window_feats, session_feats, interaction_feats, last_items)\n    \n    if i == 0:\n        selected_feats_clicks = select_best_features(df_train_clicks, 'clicks', top_k=45)\n        best_features_map['clicks'] = selected_feats_clicks\n        \n    model_clicks = train_final_model(df_train_clicks, best_features_map['clicks'], 'clicks')\n    trained_models_lst[i]['clicks'] = model_clicks\n    \n    # ===================================================================\n    # GIAI ĐOẠN 2: HUẤN LUYỆN CARTS (với feature từ Clicks)\n    # ===================================================================\n    print(\"\\n--- STAGE 2: Training CARTS model with Cascade Feature ---\")\n    df_train_carts = pl.read_parquet(TEMP_CHUNK_PATH + 'training_set_carts.pqt')\n    df_train_carts = add_features(df_train_carts, time_window_feats, session_feats, interaction_feats, last_items)\n    \n    # --- Tạo Feature Cascade ---\n    print(\"  >> Creating cascade feature: 'predicted_click_score'\")\n    X_for_click_pred = df_train_carts.select(best_features_map['clicks']).to_numpy()\n    click_scores = model_clicks.predict(X_for_click_pred)\n    df_train_carts = df_train_carts.with_columns(pl.Series(\"predicted_click_score\", click_scores))\n    \n    if i == 0:\n        # Chạy Feature Selection trên dữ liệu đã có feature cascade\n        selected_feats_carts = select_best_features(df_train_carts, 'carts', top_k=45)\n        best_features_map['carts'] = selected_feats_carts\n        \n    model_carts = train_final_model(df_train_carts, best_features_map['carts'], 'carts')\n    trained_models_lst[i]['carts'] = model_carts\n\n    # ===================================================================\n    # GIAI ĐOẠN 3: HUẤN LUYỆN ORDERS (với feature từ Clicks & Carts)\n    # ===================================================================\n    print(\"\\n--- STAGE 3: Training ORDERS model with Cascade Features ---\")\n    df_train_orders = pl.read_parquet(TEMP_CHUNK_PATH + 'training_set_orders.pqt')\n    df_train_orders = add_features(df_train_orders, time_window_feats, session_feats, interaction_feats, last_items)\n\n    # --- Tạo Feature Cascade ---\n    print(\"  >> Creating cascade features: 'predicted_click_score' and 'predicted_cart_score'\")\n    # Dùng model Clicks\n    X_for_click_pred_ord = df_train_orders.select(best_features_map['clicks']).to_numpy()\n    click_scores_ord = model_clicks.predict(X_for_click_pred_ord)\n    df_train_orders = df_train_orders.with_columns(pl.Series(\"predicted_click_score\", click_scores_ord))\n    \n    # Dùng model Carts\n    X_for_cart_pred_ord = df_train_orders.select(best_features_map['carts']).to_numpy()\n    cart_scores_ord = model_carts.predict(X_for_cart_pred_ord)\n    df_train_orders = df_train_orders.with_columns(pl.Series(\"predicted_cart_score\", cart_scores_ord))\n\n    if i == 0:\n        selected_feats_orders = select_best_features(df_train_orders, 'orders', top_k=45)\n        best_features_map['orders'] = selected_feats_orders\n        \n    model_orders = train_final_model(df_train_orders, best_features_map['orders'], 'orders')\n    trained_models_lst[i]['orders'] = model_orders\n\n    # Dọn dẹp\n    del df_train_clicks, df_train_carts, df_train_orders, model_clicks, model_carts, model_orders\n    del time_window_feats, session_feats, interaction_feats, last_items\n    gc.collect()\n\nprint(\"\\nAll Cascade training finished!\")","metadata":{"_uuid":"b35a7267-d7a5-4c77-95ca-908b650de8a6","_cell_guid":"7be97632-4041-4c41-8843-b8c5f9e2e017","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T14:31:56.504668Z","iopub.execute_input":"2025-12-28T14:31:56.504985Z","iopub.status.idle":"2025-12-28T15:02:33.117100Z","shell.execute_reply.started":"2025-12-28T14:31:56.504945Z","shell.execute_reply":"2025-12-28T15:02:33.115986Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(len(seeds)):\n    TEMP_CHUNK_PATH = f'/kaggle/working/temp_candidate_chunks/sample_{i}/'\n    for pred_type in ['clicks', 'carts', 'orders']:\n        trained_models_lst[i][pred_type].booster_.save_model(TEMP_CHUNK_PATH + f'lgbm_ranker_{pred_type}.txt')","metadata":{"_uuid":"641a8ed8-5b9d-473c-8df6-546c13d5e9ec","_cell_guid":"c9b7954e-f15f-466b-a1af-8d2f200de09d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:02:33.118310Z","iopub.execute_input":"2025-12-28T15:02:33.118649Z","iopub.status.idle":"2025-12-28T15:02:33.307146Z","shell.execute_reply.started":"2025-12-28T15:02:33.118601Z","shell.execute_reply":"2025-12-28T15:02:33.306332Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Validation","metadata":{"_uuid":"00d7a4d0-bb41-40d1-9bd1-83ff1c25b313","_cell_guid":"42914314-d1bc-4fab-bb7b-7ba66fd079b5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"%%time\nfrom src.pipeline.preprocess import *\nfrom src.core.utils import *\n# Load train parquet\ntrain_df = load_raw_data_parquet('/kaggle/input/otto-chunk-data-inparquet-format/train_parquet/*')\n# Load test parquet\nvalid_df = load_raw_data_parquet('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/*')","metadata":{"_uuid":"d6224481-9537-427d-8bff-840b75d6e2da","_cell_guid":"f750f32b-bb6f-484d-921e-b81ecbdf3efe","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:02:33.307829Z","iopub.execute_input":"2025-12-28T15:02:33.308099Z","iopub.status.idle":"2025-12-28T15:03:16.110106Z","shell.execute_reply.started":"2025-12-28T15:02:33.308079Z","shell.execute_reply":"2025-12-28T15:03:16.108914Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nfrom src.candidate_generation.loader import *\nMATRIX_BASE_PATH = '/kaggle/input/otto-precal-covisit-matrices/submission/all/'\n\ndf_clicks_valid = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_clicks_*.pqt', 'clicks', 20)\ndf_buys_valid = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_carts_orders_*.pqt', 'buys', 15)\ndf_buy2buy_valid = load_covisit_matrix(MATRIX_BASE_PATH + 'top_*_buy2buy_*.pqt', 'buy2buy', 15)","metadata":{"_uuid":"6d321562-b953-4c9f-b7ba-c9c1fc0c423a","_cell_guid":"76e8c9d5-b83b-406f-90e5-08681dbe4e29","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:03:16.112038Z","iopub.execute_input":"2025-12-28T15:03:16.112310Z","iopub.status.idle":"2025-12-28T15:03:39.558604Z","shell.execute_reply.started":"2025-12-28T15:03:16.112288Z","shell.execute_reply":"2025-12-28T15:03:39.557431Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_df = valid_df.select(['session', 'aid']).unique()","metadata":{"_uuid":"e01b23e1-99c3-432d-8617-1b11ddc01ade","_cell_guid":"5d7c8177-f6b0-4731-b24d-a04718572c41","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:03:39.559851Z","iopub.execute_input":"2025-12-28T15:03:39.560150Z","iopub.status.idle":"2025-12-28T15:03:40.829144Z","shell.execute_reply.started":"2025-12-28T15:03:39.560129Z","shell.execute_reply":"2025-12-28T15:03:40.828222Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nimport polars as pl\n# Prepare global popular candidate\ntop_clicks_popular = train_df.filter(pl.col('type') == 0)['aid'].value_counts().sort(['count'], descending=[True]).head(15)['aid']\ntop_carts_popular = train_df.filter(pl.col('type') == 1)['aid'].value_counts().sort(['count'], descending=[True]).head(20)['aid']\ntop_orders_popular = train_df.filter(pl.col('type') == 2)['aid'].value_counts().sort(['count'], descending=[True]).head(20)['aid']\n\npopular_items = pl.concat([top_clicks_popular, \n                           top_carts_popular, \n                           top_orders_popular]).unique()","metadata":{"_uuid":"c8b31e59-a9c4-4a05-9af1-c5585dc03aca","_cell_guid":"ed89d19f-0fd4-4f4f-9574-9e4c5024f9c5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:03:40.830185Z","iopub.execute_input":"2025-12-28T15:03:40.830885Z","iopub.status.idle":"2025-12-28T15:05:02.835989Z","shell.execute_reply.started":"2025-12-28T15:03:40.830859Z","shell.execute_reply":"2025-12-28T15:05:02.834965Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"popular_items_df = pl.DataFrame({'candidate_aid': popular_items})\npopular_items_df","metadata":{"_uuid":"73b3a9b7-4cd3-44a5-b4f9-0b38aa00f541","_cell_guid":"ad174c58-42db-4ebe-a3ee-ca971b9dbd50","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:05:02.836960Z","iopub.execute_input":"2025-12-28T15:05:02.837279Z","iopub.status.idle":"2025-12-28T15:05:02.857363Z","shell.execute_reply.started":"2025-12-28T15:05:02.837250Z","shell.execute_reply":"2025-12-28T15:05:02.856397Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"item_popularity_df = pre_compute_item_popularity(train_df)","metadata":{"_uuid":"beec9c66-df89-4b21-9e3b-931fc41e47f7","_cell_guid":"0ba612a8-67e4-498d-a6fd-1f6984c9bc1b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:05:02.858684Z","iopub.execute_input":"2025-12-28T15:05:02.859051Z","iopub.status.idle":"2025-12-28T15:06:11.399196Z","shell.execute_reply.started":"2025-12-28T15:05:02.859022Z","shell.execute_reply":"2025-12-28T15:06:11.398128Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_window_feats = [\n    create_time_window_features(train_df, None),\n    create_time_window_features(train_df, 7)\n]\nsession_feats, interaction_feats, last_items = create_session_context_features(valid_df)","metadata":{"_uuid":"b5e93b2a-9aa8-4a1d-924e-396f7e8e05f7","_cell_guid":"c7ec733c-cc81-4d93-aa04-b572fce474fb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:06:11.400416Z","iopub.execute_input":"2025-12-28T15:06:11.400792Z","iopub.status.idle":"2025-12-28T15:06:46.056671Z","shell.execute_reply.started":"2025-12-28T15:06:11.400760Z","shell.execute_reply":"2025-12-28T15:06:46.055552Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\nN_CHUNKS = 30\n\nall_sessions = history_df['session'].unique().to_list()\nchunk_size = len(all_sessions) // N_CHUNKS\n\n\nTEMP_PREDICTION_CHUNK_PATH = '/kaggle/working/temp_prediction_chunks/' # Thư mục để lưu các file tạm\nos.makedirs(TEMP_PREDICTION_CHUNK_PATH, exist_ok=True)\n\nstats_before_filtering = []\nstats_after_filtering = []\n\nW_HISTORY = 12.0\nW_REPETITION = 2.0\nW_BUY2BUY = 0.6\nW_BUYS = 0.3\nW_CLICKS = 0.2\n\nPRE_FILTER_TOP_K = 40\nPRE_FILTER_RANDOM_N = 25\n\nprint(f\"\\n--- Processing {len(all_sessions)} sessions in {N_CHUNKS+1} chunks ---\")\nfor i in tqdm(range(N_CHUNKS + 1)):\n    start = i * chunk_size\n    end = (i + 1) * chunk_size\n    if start >= len(all_sessions):\n        break\n    \n    session_chunk_ids = all_sessions[start:end]\n    history_chunk = history_df.filter(pl.col('session').is_in(session_chunk_ids))\n    \n    # Gọi hàm xử lý cho chunk\n    chunk_result = process_chunk(history_chunk, popular_items_df,\n                                 df_clicks_valid, df_buys_valid, df_buy2buy_valid)\n    # Thêm đặc trưng cuối cùng cho nguồn popular\n    chunk_result = chunk_result.with_columns(\n        pl.col('candidate_aid').is_in(popular_items_df['candidate_aid']).cast(pl.UInt8).alias('source_popular')\n    )\n    # --- B. THÊM FEATURE CHO CHUNK ---\n    feature_chunk = add_features(\n        chunk_result, \n        time_window_feats,\n        session_feats,\n        interaction_feats,\n        last_items,\n    )\n\n    # ===================================================================\n    # BƯỚC ĐO LƯỜNG #1: TRƯỚC KHI LỌC\n    # ===================================================================\n    # Tính số lượng ứng viên cho mỗi session trong chunk này\n    count_before = feature_chunk.group_by('session').count()\n    # Lấy giá trị trung bình và thêm vào list\n    avg_before = count_before['count'].mean()\n    if avg_before is not None:\n        stats_before_filtering.append(avg_before)\n\n    # ===================================================================\n    # BƯỚC PRE-FILTERING (Tăng tốc lgb.predict)\n    # ===================================================================\n    epsilon = 1e-9 # Để tránh chia cho 0\n    feature_chunk = feature_chunk.with_columns([\n        # Chuẩn hóa num_repetitions_in_session\n        (\n            (pl.col('num_repetitions') - pl.col('num_repetitions').min().over('session')) /\n            (pl.col('num_repetitions').max().over('session') - pl.col('num_repetitions').min().over('session') + epsilon)\n        ).alias('norm_repetition'),\n        \n        # Chuẩn hóa wgt_buy2buy\n        (\n            (pl.col('wgt_buy2buy') - pl.col('wgt_buy2buy').min().over('session')) /\n            (pl.col('wgt_buy2buy').max().over('session') - pl.col('wgt_buy2buy').min().over('session') + epsilon)\n        ).alias('norm_wgt_buy2buy'),\n        \n        # Chuẩn hóa wgt_buys\n        (\n            (pl.col('wgt_buys') - pl.col('wgt_buys').min().over('session')) /\n            (pl.col('wgt_buys').max().over('session') - pl.col('wgt_buys').min().over('session') + epsilon)\n        ).alias('norm_wgt_buys'),\n        \n        # Chuẩn hóa wgt_clicks\n        (\n            (pl.col('wgt_clicks') - pl.col('wgt_clicks').min().over('session')) /\n            (pl.col('wgt_clicks').max().over('session') - pl.col('wgt_clicks').min().over('session') + epsilon)\n        ).alias('norm_wgt_clicks'),\n    ]).fill_nan(0) # Điền 0 cho các trường hợp max == min\n    \n    # --- TÍNH TOÁN ĐIỂM SỐ HEURISTIC (Sử dụng các feature đã chuẩn hóa) ---\n    feature_chunk = feature_chunk.with_columns(\n        (\n            (pl.col('source_history') * W_HISTORY) +\n            (pl.col('norm_repetition') * W_REPETITION) +\n            (pl.col('norm_wgt_buy2buy') * W_BUY2BUY) +\n            (pl.col('norm_wgt_buys') * W_BUYS) +\n            (pl.col('norm_wgt_clicks') * W_CLICKS)\n        ).alias('heuristic_score')\n    )\n\n    print(feature_chunk.to_pandas().isna().sum().sum())\n    \n    \n    # --- Sắp xếp TOÀN BỘ chunk theo session và điểm heuristic ---\n    feature_chunk_sorted = feature_chunk.sort(['session', 'heuristic_score'], descending=[False, True])\n    \n    # --- Lấy Top K (Exploitation) ---\n    # .head() sẽ lấy các dòng đầu tiên (có điểm cao nhất) cho mỗi nhóm session\n    top_k_candidates = feature_chunk_sorted.group_by('session', maintain_order=True).head(PRE_FILTER_TOP_K)\n    \n    # --- Lấy Phần còn lại ---\n    # Dùng anti_join để tìm tất cả các ứng viên không thuộc top K\n    remaining_candidates = feature_chunk.join(\n        top_k_candidates.select(['session', 'candidate_aid']),\n        on=['session', 'candidate_aid'], \n        how='anti'\n    )\n    \n        # 1. Xáo trộn (shuffle) các dòng trong mỗi nhóm session\n    # 2. Lấy N dòng đầu tiên (.head(N)) từ mỗi nhóm đã được xáo trộn\n    random_candidates = remaining_candidates.select(\n        pl.all().shuffle().over('session')\n    ).group_by('session', maintain_order=True).head(PRE_FILTER_RANDOM_N)\n\n    \n    # --- Gộp 2 phần lại ---\n    feature_chunk_filtered = pl.concat([\n        top_k_candidates,\n        random_candidates\n    ])\n    \n    # Loại bỏ các cột tạm thời\n    cols_to_drop = [col for col in feature_chunk_filtered.columns if col.startswith('norm_') or col == 'heuristic_score']\n    feature_chunk_filtered = feature_chunk_filtered.drop(cols_to_drop)\n    \n    # ===================================================================\n    # BƯỚC ĐO LƯỜNG #2: SAU KHI LỌC\n    # ===================================================================\n    count_after = feature_chunk_filtered.group_by('session').count()\n    avg_after = count_after['count'].mean()\n    if avg_after is not None:\n        stats_after_filtering.append(avg_after)\n        \n    print(f\"  Chunk {i}: Avg candidates Before={avg_before:.1f}, After={avg_after:.1f}\")\n    \n    # --- C. DỰ ĐOÁN TRÊN CHUNK (BƯỚC MỚI) ---\n    print(f\"  Chunk {i}: Predicting scores...\")\n    \n    # Sắp xếp chunk theo session để đảm bảo thứ tự\n    feature_chunk_filtered = feature_chunk_filtered.sort('session')\n    \n    # List để lưu các mảng điểm số từ mỗi bộ model\n    # --- Dự đoán Clicks trước ---\n    clicks_feats = best_features_map['clicks']\n    X_chunk_clicks = feature_chunk_filtered.select(clicks_feats).to_numpy()\n    all_scores_clicks = [model_set['clicks'].predict(X_chunk_clicks) for model_set in trained_models_lst]\n    avg_scores_clicks = np.mean(all_scores_clicks, axis=0)\n    \n    # Thêm điểm clicks vào chunk để dùng cho carts/orders\n    feature_chunk_filtered = feature_chunk_filtered.with_columns(pl.Series(\"predicted_click_score\", avg_scores_clicks))\n\n    # --- Dự đoán Carts ---\n    carts_feats = best_features_map['carts']\n    X_chunk_carts = feature_chunk_filtered.select(carts_feats).to_numpy()\n    all_scores_carts = [model_set['carts'].predict(X_chunk_carts) for model_set in trained_models_lst]\n    avg_scores_carts = np.mean(all_scores_carts, axis=0)\n\n    # Thêm điểm carts vào chunk để dùng cho orders\n    feature_chunk_filtered = feature_chunk_filtered.with_columns(pl.Series(\"predicted_cart_score\", avg_scores_carts))\n\n    # --- Dự đoán Orders ---\n    orders_feats = best_features_map['orders']\n    X_chunk_orders = feature_chunk_filtered.select(orders_feats).to_numpy()\n    all_scores_orders = [model_set['orders'].predict(X_chunk_orders) for model_set in trained_models_lst]\n    avg_scores_orders = np.mean(all_scores_orders, axis=0)\n\n    # --- D. LƯU KẾT QUẢ ---\n    prediction_chunk = feature_chunk_filtered.select(['session', 'candidate_aid']).with_columns([\n        pl.Series(\"score_clicks\", avg_scores_clicks),\n        pl.Series(\"score_carts\", avg_scores_carts),\n        pl.Series(\"score_orders\", avg_scores_orders)\n    ])\n    \n    # Lưu chunk kết quả dự đoán ra đĩa\n    prediction_chunk.write_parquet(TEMP_PREDICTION_CHUNK_PATH + f'predictions_chunk_{i}.pqt')\n    \n    del chunk_result, feature_chunk, prediction_chunk, feature_chunk_filtered, X_chunk_clicks, X_chunk_carts, X_chunk_orders\n        \n    gc.collect()","metadata":{"_uuid":"4703b31f-07d4-492d-82c7-2af399c6ea44","_cell_guid":"daa8bc99-2360-4961-b305-0b66348e3eb0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:06:46.058295Z","iopub.execute_input":"2025-12-28T15:06:46.058689Z","iopub.status.idle":"2025-12-28T15:14:34.737202Z","shell.execute_reply.started":"2025-12-28T15:06:46.058656Z","shell.execute_reply":"2025-12-28T15:14:34.734327Z"},"scrolled":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del df_clicks_valid, df_buys_valid, df_buy2buy_valid, popular_items_df\n_ = gc.collect()","metadata":{"_uuid":"7b83efe8-a1f2-497e-b85f-c15914045a40","_cell_guid":"c57f89a9-6149-4229-8613-a32eae1125ae","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:14:34.738230Z","iopub.status.idle":"2025-12-28T15:14:34.738704Z","shell.execute_reply.started":"2025-12-28T15:14:34.738463Z","shell.execute_reply":"2025-12-28T15:14:34.738482Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"--- Aggregating all prediction chunks ---\")\nTEMP_PREDICTION_CHUNK_PATH = '/kaggle/working/temp_prediction_chunks/'\n\n# Sử dụng scan_parquet để đọc tất cả các file chunk một cách \"lười biếng\"\nlazy_predictions_df = pl.scan_parquet(TEMP_PREDICTION_CHUNK_PATH + 'predictions_chunk_*.pqt')\n\n# .collect() để hiện thực hóa DataFrame. Bước này bây giờ rất nhanh và nhẹ.\npredictions_df = lazy_predictions_df.collect()\n\nprint(f\"Aggregated predictions DataFrame shape: {predictions_df.shape}\")","metadata":{"_uuid":"fa0af6d4-ac92-4176-81f1-b07358af4cfd","_cell_guid":"e919f720-9828-42c8-bce4-2399ebd70e24","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:14:34.740264Z","iopub.status.idle":"2025-12-28T15:14:34.740570Z","shell.execute_reply.started":"2025-12-28T15:14:34.740433Z","shell.execute_reply":"2025-12-28T15:14:34.740446Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n--- Ranking candidates and creating separate prediction dataframes ---\")\n\n# Dictionary để lưu kết quả cuối cùng\nfinal_predictions = {}\n\nfor model_type in ['clicks', 'carts', 'orders']:\n    \n    score_col = f'score_{model_type}'\n    preds_for_type = predictions_df.select(['session', 'candidate_aid', score_col])\n    \n    # Sắp xếp và lấy top 20\n    top_20_preds = preds_for_type.sort(score_col, descending=True) \\\n                                 .group_by('session', maintain_order=False) \\\n                                 .head(20) \\\n                                 .group_by('session', maintain_order=True) \\\n                                 .agg(pl.col('candidate_aid').alias('labels'))\n    \n    final_predictions[model_type] = top_20_preds\n\n# --- BƯỚC 1 & 2 (Không đổi) ---\npred_df_clicks = final_predictions['clicks'].with_columns(pl.col('session').cast(pl.Utf8) + \"_clicks\")\npred_df_carts = final_predictions['carts'].with_columns(pl.col('session').cast(pl.Utf8) + \"_carts\")\npred_df_orders = final_predictions['orders'].with_columns(pl.col('session').cast(pl.Utf8) + \"_orders\")\n\nsubmission_df = pl.concat([\n    pred_df_clicks,\n    pred_df_carts,\n    pred_df_orders\n]).rename({'session': 'session_type'})\n\n# --- BƯỚC 3: CHUYỂN ĐỔI LIST THÀNH CHUỖI (ĐÃ SỬA LỖI) ---\nsubmission_df = submission_df.with_columns(\n    # 1. Áp dụng `cast(pl.Utf8)` cho TỪNG PHẦN TỬ bên trong list\n    pl.col('labels').list.eval(pl.element().cast(pl.Utf8))\n    # 2. Bây giờ mới JOIN các chuỗi đó lại\n    .list.join(\" \")\n)\n\n# --- BƯỚC 4: LƯU RA FILE CSV (Không đổi) ---\nprint(\"Saving submission.csv...\")\nsubmission_df.write_csv(\"/kaggle/working/submission.csv\")\n\nprint(\"Submission file created successfully!\")\ndisplay(submission_df.head())","metadata":{"_uuid":"ef4d052c-53a7-4411-ba26-8afa8ed2728c","_cell_guid":"66e56b1d-ded2-4ae2-8fbe-918036c0bd9b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-28T15:14:34.741913Z","iopub.status.idle":"2025-12-28T15:14:34.742245Z","shell.execute_reply.started":"2025-12-28T15:14:34.742065Z","shell.execute_reply":"2025-12-28T15:14:34.742080Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"5ec86ec9-1497-4151-9c0c-d6a3699339fa","_cell_guid":"13d56ee0-7361-4e41-973b-0937023fe580","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}