{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":14327604,"sourceType":"datasetVersion","datasetId":9140518}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\nimport os\nimport sys\nimport copy\nfrom datetime import datetime\nimport gc\nimport pickle as pkl\nimport shelve\n\nimport pandas as pd\nimport numpy as np\nimport cudf\n\nsys.path.append(\"../input/\")\nfrom handmhelpers import io as h_io, sub as h_sub, cv as h_cv, fe as h_fe\nfrom handmhelpers import modeling as h_modeling, candidates as h_can, pairs as h_pairs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:17:56.892550Z","iopub.execute_input":"2025-12-29T05:17:56.892837Z","iopub.status.idle":"2025-12-29T05:18:01.163020Z","shell.execute_reply.started":"2025-12-29T05:17:56.892806Z","shell.execute_reply":"2025-12-29T05:18:01.162326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load and convert data","metadata":{}},{"cell_type":"code","source":"from datetime import timedelta\nimport cudf\nimport numpy as np\n\ndef patched_day_week_numbers(dates: cudf.Series):\n    pd_dates = cudf.to_datetime(dates)\n    unique_dates = cudf.Series(pd_dates.unique())\n    numbered_days = unique_dates - unique_dates.min() + timedelta(1)\n    numbered_days = numbered_days.dt.days\n    extra_days = numbered_days.max() % 7\n    numbered_days -= extra_days\n    day_weeks = (numbered_days + 6) // 7  # không dùng applymap\n    day_weeks_map = cudf.DataFrame({\"day_weeks\": day_weeks, \"unique_dates\": unique_dates}).set_index(\"unique_dates\")[\"day_weeks\"]\n    all_day_weeks = pd_dates.map(day_weeks_map).astype(\"int8\")\n    return all_day_weeks\n\nimport handmhelpers.fe as h_fe\nh_fe.day_week_numbers = patched_day_week_numbers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:01.165240Z","iopub.execute_input":"2025-12-29T05:18:01.165512Z","iopub.status.idle":"2025-12-29T05:18:01.171533Z","shell.execute_reply.started":"2025-12-29T05:18:01.165463Z","shell.execute_reply":"2025-12-29T05:18:01.170813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nc, t, a = h_io.load_data(files=['customers.csv', 'transactions_train.csv', 'articles.csv'])        \n\nindex_to_id_dict_path = h_fe.reduce_customer_id_memory(c, [t])\nt[\"week_number\"] = h_fe.day_week_numbers(t[\"t_dat\"])\nt[\"t_dat\"] = h_fe.day_numbers(t[\"t_dat\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:01.172407Z","iopub.execute_input":"2025-12-29T05:18:01.172774Z","iopub.status.idle":"2025-12-29T05:18:05.020149Z","shell.execute_reply.started":"2025-12-29T05:18:01.172738Z","shell.execute_reply":"2025-12-29T05:18:05.019365Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Get item pairs","metadata":{}},{"cell_type":"code","source":"%%time\n# Tạo cặp bài viết cho nhiều tuần lịch sử (94–104) với 15 cặp mỗi bài để nâng recall ứng viên\npairs_per_item = 15\n\nweek_number_pairs = {}\nfor week_number in [94,95,96, 97, 98, 99, 100, 101, 102, 103, 104]:\n    print(f\"Creating pairs for week number {week_number}\")\n    week_number_pairs[week_number] = h_pairs.create_pairs(\n        t, week_number, pairs_per_item, verbose=False\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:05.021209Z","iopub.execute_input":"2025-12-29T05:18:05.021752Z","iopub.status.idle":"2025-12-29T05:18:44.559656Z","shell.execute_reply.started":"2025-12-29T05:18:05.021727Z","shell.execute_reply":"2025-12-29T05:18:44.558935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Main retrieval/features function!","metadata":{}},{"cell_type":"code","source":"def create_candidates_with_features_df(t, c, a, customer_batch=None, **kwargs):\n    # Tách dữ liệu theo tuần label: features dùng các tuần trước, label là tuần đích\n    features_df, label_df = h_cv.feature_label_split(\n        t, kwargs[\"label_week\"], kwargs[\"feature_periods\"]\n    )\n    # Chuẩn hóa thời gian về “cách đây bao nhiêu ngày/tuần”\n    features_df[\"t_dat\"] = h_fe.how_many_ago(features_df[\"t_dat\"])\n    features_df[\"week_number\"] = h_fe.how_many_ago(features_df[\"week_number\"])\n\n    article_pairs_df = week_number_pairs[kwargs[\"label_week\"] - 1]\n\n    # Xác định tập khách hàng cần xử lý (full/batch/label)\n    if len(label_df) > 0:\n        customers = label_df[\"customer_id\"].unique()\n    elif customer_batch is not None:\n        customers = customer_batch\n    else:\n        customers = None\n\n    # ----- Ứng viên từ nhiều nguồn và đặc trưng rule -----\n    features_db = {}\n    recent_customer_cand, features_db[\"customer_article\"] = h_can.create_recent_customer_candidates(\n        features_df, kwargs[\"ca_num_weeks\"], customers=customers\n    )\n    (\n        cust_last_week_cand,\n        cust_last_week_pair_cand,\n        features_db[\"clw\"],\n        features_db[\"clw_pairs\"],\n    ) = h_can.create_last_customer_weeks_and_pairs(\n        features_df, article_pairs_df, kwargs[\"clw_num_weeks\"], kwargs[\"clw_num_pair_weeks\"], customers=customers\n    )\n    popular_can, features_db[\"popular_articles\"] = h_can.create_popular_article_cand(\n        features_df, c, a, kwargs[\"pa_num_weeks\"], kwargs[\"hier_col\"],\n        num_candidates=kwargs[\"num_recent_candidates\"],\n        num_articles=kwargs[\"num_recent_articles\"],\n        customers=customers,\n    )\n    (age_bucket_can, _, age_bucket_pair_features) = h_can.create_age_bucket_candidates(\n        features_df, c, kwargs[\"num_age_buckets\"], articles=kwargs[\"num_recent_articles\"], customers=customers\n    )\n    features_db[\"age_bucket\"] = age_bucket_pair_features\n\n    # NEW: ứng viên random walk\n    random_walk_cand, features_db[\"f_random_walk\"] = h_can.create_random_walk_candidates(\n        features_df,\n        article_pairs_df,\n        seed_weeks=kwargs[\"gd_seed_weeks\"],\n        seed_articles=kwargs[\"gd_seed_articles\"],\n        num_steps=kwargs[\"gd_steps\"],\n        restart_prob=kwargs[\"gd_restart_prob\"],\n        topk=kwargs[\"gd_topk\"],\n        weight_col=kwargs[\"gd_weight_col\"],\n        recency_weight=kwargs[\"gd_recency_weight\"],\n        exclude_seed_items=kwargs[\"gd_exclude_seed\"],\n        customers=customers,\n    )\n\n    # Gom rule-score từ từng nguồn\n    def build_rule_part(cand_df, feature_tuple, score_col, rule_name):\n        feature_df = feature_tuple[1].reset_index()[[\"customer_id\", \"article_id\", score_col]]\n        tmp = cand_df.merge(feature_df, on=[\"customer_id\", \"article_id\"], how=\"left\")\n        tmp = tmp.rename(columns={score_col: \"rule_score\"})\n        tmp[\"rule_score\"] = tmp[\"rule_score\"].fillna(-1)\n        tmp[\"rule\"] = rule_name\n        return tmp[[\"customer_id\", \"article_id\", \"rule\", \"rule_score\"]]\n\n    rule_parts = [\n        build_rule_part(recent_customer_cand, features_db[\"customer_article\"], \"ca_purchase_count\", \"recent\"),\n        build_rule_part(cust_last_week_cand, features_db[\"clw\"], \"ca_count\", \"last_weeks\"),\n        build_rule_part(cust_last_week_pair_cand, features_db[\"clw_pairs\"], \"pair_lift\", \"pairs\"),\n        build_rule_part(age_bucket_can, features_db[\"age_bucket\"], \"article_bucket_count\", \"age_bucket\"),\n    ]\n    # rule popular\n    pop_feature_df = features_db[\"popular_articles\"][1].reset_index()[[\"article_id\", \"recent_popularity_counts\"]]\n    pop_rule = popular_can.merge(pop_feature_df, on=\"article_id\", how=\"left\")\n    pop_rule = pop_rule.rename(columns={\"recent_popularity_counts\": \"rule_score\"})\n    pop_rule[\"rule_score\"] = pop_rule[\"rule_score\"].fillna(-1)\n    pop_rule[\"rule\"] = \"popular\"\n    rule_parts.append(pop_rule[[\"customer_id\", \"article_id\", \"rule\", \"rule_score\"]])\n    # rule random_walk\n    rule_parts.append(\n        build_rule_part(random_walk_cand, features_db[\"f_random_walk\"], \"rw_score\", \"random_walk\")\n    )\n\n    rule_df = cudf.concat(rule_parts).sort_values(\n        [\"rule\", \"customer_id\", \"rule_score\"], ascending=[True, True, False]\n    )\n    rule_df[\"rank_within_rule\"] = rule_df.groupby([\"rule\", \"customer_id\"]).cumcount()\n\n    rule_features_df = (\n        rule_df.groupby([\"customer_id\", \"article_id\"])\n        .agg({\"rule\": \"nunique\", \"rule_score\": \"max\", \"rank_within_rule\": \"min\"})\n        .reset_index()\n    )\n    rule_features_df.columns = [\"customer_id\", \"article_id\", \"n_sources\", \"best_rule_score\", \"best_rank_within_rule\"]\n\n    # Thêm cờ nguồn\n    for rule_name in rule_df[\"rule\"].unique().to_pandas():\n        flag_df = rule_df[rule_df[\"rule\"] == rule_name][[\"customer_id\", \"article_id\"]].drop_duplicates()\n        flag_df[f\"{rule_name}_flag\"] = 1\n        rule_features_df = rule_features_df.merge(flag_df, how=\"left\", on=[\"customer_id\", \"article_id\"])\n        rule_features_df[f\"{rule_name}_flag\"] = rule_features_df[f\"{rule_name}_flag\"].fillna(0).astype(\"int8\")\n\n    # Hợp nhất ứng viên (đã thêm random_walk)\n    cand = cudf.concat([\n        popular_can,\n        recent_customer_cand,\n        cust_last_week_cand,\n        cust_last_week_pair_cand,\n        age_bucket_can,\n        random_walk_cand,\n    ]).drop_duplicates().sort_values([\"customer_id\", \"article_id\"]).reset_index(drop=True)\n    del popular_can, recent_customer_cand, cust_last_week_cand, cust_last_week_pair_cand, age_bucket_can, random_walk_cand\n\n    cand = h_can.filter_candidates(cand, t, **kwargs)\n\n    # ----- Sinh thêm đặc trưng -----\n    h_fe.create_cust_hier_features(features_df, a, kwargs[\"hier_cols\"], features_db)\n    h_fe.create_cust_hier_decay_features(features_df, a, kwargs[\"hier_cols\"], features_db,\n                                         decay_gamma=kwargs.get(\"hier_decay_gamma\", 0.3))\n    for k, v in list(features_db.items()):\n        if k.endswith(\"_decay_features\"):\n            hier = k[len(\"cust_\"):-len(\"_decay_features\")]\n            cols, df = v\n            df = df.rename(columns={\"last_seen_category_weeks_ago\": f\"last_seen_{hier}_weeks_ago\"})\n            features_db[k] = (cols, df)\n\n    h_fe.create_price_features(features_df, features_db)\n    h_fe.create_cust_features(c, features_db)\n    h_fe.create_article_cust_features(features_df, c, features_db)\n    h_fe.create_lag_features(features_df, a, kwargs[\"lag_days\"], features_db)\n    h_fe.create_rebuy_features(features_df, features_db)\n    h_fe.create_cust_t_features(features_df, a, features_db)\n    try:\n        h_fe.create_art_t_features(features_df, a, features_db)\n    except TypeError:\n        h_fe.create_art_t_features(features_df, features_db)\n    del features_df\n\n    if customers is not None:\n        cand = cand[cand[\"customer_id\"].isin(customers)]\n\n    if kwargs[\"cv\"]:\n        ground_truth_candidates = label_df[[\"customer_id\", \"article_id\"]].drop_duplicates()\n        h_cv.report_candidates(cand, ground_truth_candidates)\n        del ground_truth_candidates\n\n    cand_with_f_df = h_can.add_features_to_candidates(cand, features_db, c, a)\n    cand_with_f_df = cand_with_f_df.merge(rule_features_df, how=\"left\", on=[\"customer_id\", \"article_id\"])\n\n    for article_col in kwargs[\"article_columns\"]:\n        art_col_map = a.set_index(\"article_id\")[article_col]\n        cand_with_f_df[article_col] = cand_with_f_df[\"article_id\"].map(art_col_map)\n\n    # Fill giá trị rule cho ứng viên mới (có random_walk_flag)\n    rule_fill = {\n        \"n_sources\": 0,\n        \"best_rule_score\": -1,\n        \"best_rank_within_rule\": 127,\n        \"recent_flag\": 0,\n        \"last_weeks_flag\": 0,\n        \"pairs_flag\": 0,\n        \"age_bucket_flag\": 0,\n        \"popular_flag\": 0,\n        \"random_walk_flag\": 0,\n    }\n    cand_with_f_df = cand_with_f_df.fillna(rule_fill)\n\n    # Target encode các cột category dựa trên best_rule_score (nếu có)\n    target_col = \"best_rule_score\" if \"best_rule_score\" in cand_with_f_df.columns else None\n    for col in cand_with_f_df.columns:\n        if col in [\"customer_id\", \"article_id\"]:\n            continue\n        if str(cand_with_f_df[col].dtype) not in [\"int8\",\"int16\",\"int32\",\"int64\",\"float16\",\"float32\",\"float64\",\"bool\"]:\n            if target_col:\n                te = cand_with_f_df.groupby(col)[target_col].mean()\n                cand_with_f_df[col] = cand_with_f_df[col].map(te).fillna(0).astype(\"float32\")\n            else:\n                cand_with_f_df[col] = cand_with_f_df[col].astype(\"category\").cat.codes.astype(\"float32\")\n\n    if kwargs[\"selected_features\"] is not None:\n        cand_with_f_df = cand_with_f_df[[\"customer_id\", \"article_id\"] + kwargs[\"selected_features\"]]\n\n    assert len(cand) == len(cand_with_f_df), \"seem to have duplicates in the feature dfs\"\n    del cand\n\n    return cand_with_f_df, label_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:44.560596Z","iopub.execute_input":"2025-12-29T05:18:44.560828Z","iopub.status.idle":"2025-12-29T05:18:44.585579Z","shell.execute_reply.started":"2025-12-29T05:18:44.560806Z","shell.execute_reply":"2025-12-29T05:18:44.584628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_model_score(ids_df, preds, truth_df):\n    predictions = h_modeling.create_predictions(ids_df, preds)\n    true_labels = h_cv.ground_truth(truth_df).set_index(\"customer_id\")[\"prediction\"]\n    score = round(h_cv.comp_average_precision(true_labels, predictions),5)\n    \n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:44.586641Z","iopub.execute_input":"2025-12-29T05:18:44.586957Z","iopub.status.idle":"2025-12-29T05:18:44.606175Z","shell.execute_reply.started":"2025-12-29T05:18:44.586923Z","shell.execute_reply":"2025-12-29T05:18:44.605552Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Parameters - one place for all!","metadata":{}},{"cell_type":"code","source":"# Cấu hình chạy CV (đa tuần, tăng ứng viên/estimators + random walk)\ncv_params = {\n    \"cv\": True,                     # bật báo cáo recall ứng viên\n    \"feature_periods\": 105,         # dùng 105 tuần lịch sử cho feature\n    \"label_week\": 104,              # tuần label mặc định, sẽ bị override bởi cv_weeks\n    \"index_to_id_dict_path\": index_to_id_dict_path,\n    \"pairs_file_version\": \"_v3_5_ex\",\n    \"num_recent_candidates\": 80,    # tăng số ứng viên recent để nâng recall\n    \"num_recent_articles\": 20,      # tăng bài phổ biến/age bucket\n    \"hier_col\": \"department_no\",\n    \"ca_num_weeks\": 3,\n    \"clw_num_weeks\": 12,\n    \"clw_num_pair_weeks\": 2,\n    \"pa_num_weeks\": 2,              # kéo dài phổ biến thêm 2 tuần\n    \"num_age_buckets\": 4,\n    \"filter_recent_art_weeks\": 1,\n    \"filter_num_articles\": None,\n    \"lag_days\": [1, 3, 7, 14, 28],\n    \"article_columns\": [\"index_code\"],\n    \"hier_cols\": [\n        \"department_no\", \"section_no\", \"index_group_no\", \"index_code\",\n        \"product_type_no\", \"product_group_name\"\n    ],\n    \"hier_decay_gamma\": 0.3,\n    \"selected_features\": None,\n    # random walk with restart candidates\n    \"gd_seed_weeks\": 12,\n    \"gd_seed_articles\": 12,\n    \"gd_steps\": 2,\n    \"gd_restart_prob\": \"adaptive\",\n    \"gd_topk\": 24,\n    \"gd_weight_col\": \"customer_count\",\n    \"gd_recency_weight\": True,\n    \"gd_exclude_seed\": True,\n    \"lgbm_params\": {                # tăng capacity model cho CV\n        \"n_estimators\": 400,\n        \"learning_rate\": 0.05,\n        \"num_leaves\": 64,\n        \"feature_fraction\": 0.8,\n        \"bagging_fraction\": 0.8,\n        \"bagging_freq\": 1,\n    },\n    \"log_evaluation\": 10,\n    \"early_stopping\": 30,\n    \"eval_at\": 12,\n    \"save_model\": True,\n    \"num_concats\": 5,               # ghép 5 tuần train để tăng dữ liệu\n}\n\n# Cấu hình train/predict submit (n_estimators cao hơn, ensemble 2 model + random walk)\nsub_params = {\n    \"cv\": False,\n    \"feature_periods\": 105,\n    \"label_week\": 105,\n    \"index_to_id_dict_path\": index_to_id_dict_path,\n    \"pairs_file_version\": \"_v3_5_ex\",\n    \"num_recent_candidates\": 80,\n    \"num_recent_articles\": 20,\n    \"hier_col\": \"department_no\",\n    \"ca_num_weeks\": 3,\n    \"clw_num_weeks\": 12,\n    \"clw_num_pair_weeks\": 2,\n    \"pa_num_weeks\": 2,\n    \"num_age_buckets\": 4,\n    \"filter_recent_art_weeks\": 1,\n    \"filter_num_articles\": None,\n    \"lag_days\": [1, 3, 7, 14, 28],\n    \"article_columns\": [\"index_code\"],\n    \"hier_cols\": [\n        \"department_no\", \"section_no\", \"index_group_no\", \"index_code\",\n        \"product_type_no\", \"product_group_name\"\n    ],\n    \"hier_decay_gamma\": 0.3,\n    \"selected_features\": None,\n    # random walk with restart candidates\n    \"gd_seed_weeks\": 12,\n    \"gd_seed_articles\": 12,\n    \"gd_steps\": 2,\n    \"gd_restart_prob\": \"adaptive\",\n    \"gd_topk\": 24,\n    \"gd_weight_col\": \"customer_count\",\n    \"gd_recency_weight\": True,\n    \"gd_exclude_seed\": True,\n    \"lgbm_params\": {                # nhiều cây hơn cho submit\n        \"n_estimators\": 500,\n        \"learning_rate\": 0.05,\n        \"num_leaves\": 64,\n        \"feature_fraction\": 0.8,\n        \"bagging_fraction\": 0.8,\n        \"bagging_freq\": 1,\n    },\n    \"log_evaluation\": 10,\n    \"eval_at\": 12,\n    \"prediction_models\": [\"model_104\", \"model_105\"],  # ensemble 2 model\n    \"save_model\": True,\n    \"num_concats\": 5,\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:44.607121Z","iopub.execute_input":"2025-12-29T05:18:44.607536Z","iopub.status.idle":"2025-12-29T05:18:44.621883Z","shell.execute_reply.started":"2025-12-29T05:18:44.607505Z","shell.execute_reply":"2025-12-29T05:18:44.621162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cand_features_func = create_candidates_with_features_df\nscoring_func = calculate_model_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:44.622822Z","iopub.execute_input":"2025-12-29T05:18:44.623112Z","iopub.status.idle":"2025-12-29T05:18:44.642741Z","shell.execute_reply.started":"2025-12-29T05:18:44.623081Z","shell.execute_reply":"2025-12-29T05:18:44.642020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# Chạy cross-validation cho các tuần 102–104 (đa tuần để ổn định hơn so với 1 tuần)\ncv_weeks = [102, 103, 104]\nresults = h_modeling.run_all_cvs(\n    t, c, a, cand_features_func, scoring_func,\n    cv_weeks=cv_weeks, **cv_params\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:18:44.643641Z","iopub.execute_input":"2025-12-29T05:18:44.643823Z","iopub.status.idle":"2025-12-29T06:10:52.112229Z","shell.execute_reply.started":"2025-12-29T05:18:44.643805Z","shell.execute_reply":"2025-12-29T06:10:52.111348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from cuml.fil import ForestInference as _FI\n\n_real_load = _FI.load\n\ndef _load_compat(*args, output_class=None, is_classifier=None, **kwargs):\n    # helper cũ truyền output_class -> map sang is_classifier\n    if is_classifier is None and output_class is not None:\n        is_classifier = output_class\n    return _real_load(*args, is_classifier=is_classifier, **kwargs)\n\n_FI.load = staticmethod(_load_compat)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:45:39.568367Z","iopub.execute_input":"2025-12-29T06:45:39.568930Z","iopub.status.idle":"2025-12-29T06:45:39.573259Z","shell.execute_reply.started":"2025-12-29T06:45:39.568905Z","shell.execute_reply":"2025-12-29T06:45:39.572494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\n    \"ignore\",\n    message=r\".*Parameter `output_class` was deprecated.*\",\n    category=FutureWarning,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:45:39.574257Z","iopub.execute_input":"2025-12-29T06:45:39.574642Z","iopub.status.idle":"2025-12-29T06:45:39.588536Z","shell.execute_reply.started":"2025-12-29T06:45:39.574602Z","shell.execute_reply":"2025-12-29T06:45:39.587858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\nimport handmhelpers.modeling as h_modeling\n\ndef full_sub_predict_run_small_batches(t, c, a, cand_features_func, batch_splits=8, **kwargs):\n    # Chia khách hàng thành nhiều batch nhỏ để tránh OOM khi tạo feature/predict\n    customer_batches = []\n    n = len(c)\n    for i in range(batch_splits):\n        start = i * n // batch_splits\n        end = (i + 1) * n // batch_splits\n        customer_batches.append(c[start:end][\"customer_id\"].to_pandas().to_list())\n\n    batch_preds = []\n    for idx, customer_batch in enumerate(customer_batches):\n        print(f\"generating candidates/features for batch #{idx+1} of {len(customer_batches)}\")\n        sub_ids_df, sub_X = h_modeling.prepare_prediction_dfs(\n            t, c, a, cand_features_func, customer_batch=customer_batch, **kwargs\n        )\n        print(f\"candidate/features shape of batch: ({sub_X.shape[0]:,}, {sub_X.shape[1]})\")\n\n        # Ensemble các model trong prediction_models, trung bình điểm\n        model_paths = kwargs.get(\"prediction_models\")\n        model_nums = len(model_paths)\n        first_model = h_modeling.ForestInference.load(model_paths[0], model_type=\"lightgbm\", output_class=False)\n        sub_pred = h_modeling.pred_in_batches(first_model, sub_X) / model_nums\n        del first_model\n\n        for mp in model_paths[1:]:\n            m = h_modeling.ForestInference.load(mp, model_type=\"lightgbm\", output_class=False)\n            sub_pred += h_modeling.pred_in_batches(m, sub_X) / model_nums\n            del m\n\n        batch_preds.append(h_modeling.create_predictions(sub_ids_df, sub_pred))\n        del sub_ids_df, sub_X, sub_pred\n\n    return cudf.concat(batch_preds)\n\n# Ghi đè hàm predict gốc để dùng bản chia batch nhỏ\nh_modeling.full_sub_predict_run = full_sub_predict_run_small_batches","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:10:52.140630Z","iopub.execute_input":"2025-12-29T06:10:52.140954Z","iopub.status.idle":"2025-12-29T06:10:52.157453Z","shell.execute_reply.started":"2025-12-29T06:10:52.140926Z","shell.execute_reply":"2025-12-29T06:10:52.156643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ngc.collect()\n# Train full dữ liệu submit và lưu model theo sub_params\nh_modeling.full_sub_train_run(t, c, a, cand_features_func, scoring_func, **sub_params)\n# Predict theo batch nhỏ (batch_splits=8) để tránh OOM, dùng ensemble model_104/model_105\npredictions = h_modeling.full_sub_predict_run(\n    t, c, a, cand_features_func, batch_splits=8, **sub_params\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:10:52.159976Z","iopub.execute_input":"2025-12-29T06:10:52.160275Z","iopub.status.idle":"2025-12-29T06:45:03.061181Z","shell.execute_reply.started":"2025-12-29T06:10:52.160252Z","shell.execute_reply":"2025-12-29T06:45:03.060323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = h_sub.create_sub(c[\"customer_id\"], predictions, index_to_id_dict_path)\nsub.to_csv('dev_submission.csv', index=False)\n\ndisplay(sub.head())\nprint(sub.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T06:45:03.062233Z","iopub.execute_input":"2025-12-29T06:45:03.062637Z","iopub.status.idle":"2025-12-29T06:45:39.564141Z","shell.execute_reply.started":"2025-12-29T06:45:03.062610Z","shell.execute_reply":"2025-12-29T06:45:39.563552Z"}},"outputs":[],"execution_count":null}]}