{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":14112508,"sourceType":"datasetVersion","datasetId":8989713},{"sourceId":14241442,"sourceType":"datasetVersion","datasetId":9085906},{"sourceId":14258748,"sourceType":"datasetVersion","datasetId":9098345},{"sourceId":14317728,"sourceType":"datasetVersion","datasetId":9126526}],"dockerImageVersionId":31194,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport re\nfrom datetime import timedelta\nimport traceback\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport pickle\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T17:35:05.112309Z","iopub.execute_input":"2026-01-04T17:35:05.112904Z","iopub.status.idle":"2026-01-04T17:35:07.340964Z","shell.execute_reply.started":"2026-01-04T17:35:05.112872Z","shell.execute_reply":"2026-01-04T17:35:07.339934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- CẤU HÌNH CONFIG ---\nclass Config:\n    # 1. Đường dẫn dataset gốc\n    RAW_DATA_DIR = '/kaggle/input/h-and-m-personalized-fashion-recommendations/'\n    \n    # 2. Đường dẫn dataset Candidate\n    CANDIDATE_DIR = '/kaggle/input/h-and-m-recall-model-dataset/'\n    \n    # 3. Thư mục xuất file kết quả\n    OUTPUT_DIR = '/kaggle/working/enriched_data/'\n    \n    # 4. Các Feature CẦN LOẠI BỎ\n    DROP_FEATURES = ['dssm_similarity', 'yt_similarity', 'wv_similarity', 'label']\n    \n    # 5. Kích thước chunk\n    CHUNK_SIZE = 200_000 # Giảm xuống 200k cho an toàn RAM\n\n    MAPPING_DIR = \"/kaggle/input/mapping/index_id_map/\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T17:35:07.342344Z","iopub.execute_input":"2026-01-04T17:35:07.342773Z","iopub.status.idle":"2026-01-04T17:35:07.349049Z","shell.execute_reply.started":"2026-01-04T17:35:07.342750Z","shell.execute_reply":"2026-01-04T17:35:07.347859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# 1. UTILS\n# =============================================================================\ndef reduce_mem_usage(df):\n    \"\"\"Giảm dung lượng RAM, bỏ qua cột datetime\"\"\"\n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object and not np.issubdtype(col_type, np.datetime64):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float32)\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T17:35:07.350018Z","iopub.execute_input":"2026-01-04T17:35:07.350373Z","iopub.status.idle":"2026-01-04T17:35:07.374715Z","shell.execute_reply.started":"2026-01-04T17:35:07.350344Z","shell.execute_reply":"2026-01-04T17:35:07.373635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from typing import Tuple\ndef calc_valid_date(week_num: int, last_date: str = \"2020-09-29\") -> Tuple[str]:\n    \"\"\"Calculate start and end date of a given week number.\n\n    Parameters\n    ----------\n    week_num : int\n        Week number.\n    last_date : str, optional\n        The last day, by default ``\"2020-09-22\"``.\n\n    Returns\n    -------\n    Tuple[str]\n        Start and end date of the given week number.\n    \"\"\"\n    end_date = pd.to_datetime(last_date) - pd.Timedelta(days=7 * week_num - 1)\n    start_date = end_date - pd.Timedelta(days=7)\n\n    end_date = end_date.strftime(\"%Y-%m-%d\")\n    start_date = start_date.strftime(\"%Y-%m-%d\")\n    return start_date, end_date\n\ndef calc_valid_week(dat_t: str, last_date: str = '2020-09-29') -> int:\n    week_num = (pd.to_datetime(last_date) - pd.to_datetime(dat_t)) // 7\n    return week_num","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T17:35:07.376718Z","iopub.execute_input":"2026-01-04T17:35:07.377016Z","iopub.status.idle":"2026-01-04T17:35:07.393839Z","shell.execute_reply.started":"2026-01-04T17:35:07.376993Z","shell.execute_reply":"2026-01-04T17:35:07.392754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_path = Config.RAW_DATA_DIR\npqt_path = Config.CANDIDATE_DIR\noutput_dir = Config.OUTPUT_DIR\nmapping_dir = Config.MAPPING_DIR\nprint(\"\\n=== [1] PREPARING RESOURCES ===\")\n\n# --- A. LOAD CUSTOMERS ---\nprint(\"-> Loading Customers...\")\ncustomers = pd.read_csv(raw_path + 'customers.csv')\nwith open(mapping_dir + 'user_id2index.pkl', \"rb\") as f:\n    cust_id_map = pickle.load(f)\n\ncustomers['customer_id'] = customers.index.astype('int32')\ncustomers['age'] = customers['age'].fillna(customers['age'].mean()).astype(np.int8)\ndel customers; gc.collect()\n\n# --- B. LOAD ARTICLES ---\nprint(\"-> Loading Articles...\")\narticles = pd.read_csv(raw_path + 'articles.csv', dtype={'article_id': str})\nwith open(mapping_dir + 'item_id2index.pkl', \"rb\") as f:\n    article_id_map = pickle.load(f)\ndel articles; gc.collect()\n\n# --- C. LOAD TRANSACTIONS ---\nprint(\"-> Loading Transactions (Last 5 weeks)...\")\ndf_trans = pd.read_csv(raw_path + 'transactions_train.csv')\n\nprint(\"   Mapping IDs...\")\ndf_trans['customer_id'] = df_trans['customer_id'].map(cust_id_map).fillna(-1).astype('int32')\ndf_trans['article_id'] = df_trans['article_id'].map(article_id_map).fillna(-1).astype('int32')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T17:35:07.395042Z","iopub.execute_input":"2026-01-04T17:35:07.395525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"last_date = pd.to_datetime('2020-09-29')\n\ndf_trans['week'] = (\n    last_date - pd.to_datetime(df_trans['t_dat'])\n).dt.days // 7\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_trans","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_rolling_avg_week(\n    df,\n    group_col,\n    week_col='week',\n    value_col='user_avg_spend_1w',\n    windows=(2, 3, 4, 6),\n    prefix='avg'\n):\n    df = df.copy()\n\n    # QUAN TRỌNG: week nhỏ = gần → sort DESC\n    df = df.sort_values([group_col, week_col], ascending=[True, False])\n\n    for w in windows:\n        col = f'{prefix}_{w}w'\n        df[col] = (\n            df\n            .groupby(group_col)[value_col]\n            .rolling(window=w, min_periods=1)\n            .mean()\n            .reset_index(level=0, drop=True)\n        )\n\n    return df\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nprint(\"-> Calculating Helpers...\")\n\navg_price_week = df_trans.groupby('week')['price'].mean().reset_index(name='avg_price_1w').sort_values('week', ascending=False) \n\nfor w in [2, 4]:\n    avg_price_week[f'avg_price_{w}w'] = (\n        avg_price_week['avg_price_1w']\n        .rolling(window=w, min_periods=1)\n        .mean()\n    )\n\n\n# df_trans = df_trans.merge(\n#     avg_price_week,\n#     on=['week'],\n#     how='left'\n# )\n\nuser_avg_spend_1w = (\n    df_trans\n    .groupby(['customer_id', 'week'], as_index=False)\n    .agg(user_avg_spend_1w=('price', 'mean'))\n    .sort_values(['customer_id', 'week'])\n)\n\n\n\nitem_avg_price_1w = (\n    df_trans\n    .groupby(['article_id', 'week'], as_index=False)\n    .agg(item_avg_price_1w=('price', 'mean'))\n    .sort_values(['article_id', 'week'])\n)\n\nuser_week_price = add_rolling_avg_week(\n    user_avg_spend_1w,\n    group_col='customer_id',\n    windows=(2, 4),\n    prefix='user_avg_spend'\n)\n\nuser_week_price = user_week_price.merge(avg_price_week, on=['week'], how='left')\n\nitem_week_price = add_rolling_avg_week(\n    item_avg_price_1w,\n    group_col='article_id',\n    windows=(2, 4),\n    value_col='item_avg_price_1w',\n    prefix='item_avg_price'\n)\n\nitem_week_price = item_week_price.merge(avg_price_week, on=['week'], how='left')\n\n\n# df_trans = df_trans.merge(\n#     user_week_price,\n#     on=['customer_id', 'week'],\n#     how='left'\n# )\n\n# df_trans = df_trans.merge(\n#     item_week_price,\n#     on=['article_id', 'week'],\n#     how='left'\n# ) \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in [1, 2, 4]:\n    user_week_price[f'diff_user_avg_spend_{i}w'] = user_week_price[f'user_avg_spend_{i}w'] - user_week_price[f'avg_price_{i}w']\n    item_week_price[f'diff_item_avg_spend_{i}w'] = item_week_price[f'item_avg_price_{i}w'] - item_week_price[f'avg_price_{i}w']\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_trans = reduce_mem_usage(df_trans)\nuser_week_price = reduce_mem_usage(user_week_price)\nitem_week_price = reduce_mem_usage(item_week_price)\navg_price_week = reduce_mem_usage(avg_price_week)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"])\ndf_trans['year'] = df_trans['t_dat'].dt.year\ndf_period = df_trans[\n    (df_trans['t_dat'].dt.month > 1) |\n    ((df_trans['t_dat'].dt.month == 1) & (df_trans['t_dat'].dt.day >= 1))\n]\n\ndf_period = df_period[\n    (df_period['t_dat'].dt.month < 9)\n]\nyearly_avg = (\n    df_period\n    .groupby('year', as_index=False)\n    .agg(\n        avg_price=('price', 'mean'),\n        cnt=('price', 'count')\n    )\n    .sort_values('year')\n)\navg_2019 = yearly_avg.loc[yearly_avg['year'] == 2019, 'avg_price'].values[0]\navg_2020 = yearly_avg.loc[yearly_avg['year'] == 2020, 'avg_price'].values[0]\n\ndiff_2020_2019 = avg_2020 - avg_2019\n\ndiff_2020_2019\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"avg_price_week","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pyarrow.dataset as ds\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport os\n\nfiles = os.listdir('/kaggle/input/dataset-process')\nfor f in files:\n    if 'week0_candidate' not in f: continue\n    week_df = pd.read_parquet(os.path.join('/kaggle/input/dataset-process', f) , columns=['customer_id', 'article_id'])\n    week_df['week'] = 0\n    week_last_year = avg_price_week[avg_price_week['week'] == 52].copy()\n    week_last_year['week'] = 0\n    cols = ['avg_price_1w', 'avg_price_2w', 'avg_price_4w']\n    for c in cols:\n        week_last_year[c] = week_last_year[c] + diff_2020_2019\n        \n    week_df = week_df.merge(week_last_year, on=['week'], how='left')\n    week_df = week_df.merge(\n        user_week_price.drop(\n            columns=['avg_price_1w', 'avg_price_2w', 'avg_price_4w'],\n            errors='ignore'\n        ),\n        on=['week', 'customer_id'],\n        how='left'\n    )\n    \n    week_df = week_df.merge(\n        item_week_price.drop(\n            columns=['avg_price_1w', 'avg_price_2w', 'avg_price_4w'],\n            errors='ignore'\n        ),\n        on=['week', 'article_id'],\n        how='left'\n    )    \n    for j in [1, 2, 4]:\n\n        last_item_price = (\n            item_week_price[item_week_price[f\"diff_item_avg_spend_{j}w\"].notna()]\n            .sort_values(\"week\")\n            .groupby(\"article_id\")[f\"diff_item_avg_spend_{j}w\"]\n            .first()\n        )\n    \n        week_df[f\"diff_item_avg_spend_{j}w\"] = week_df[f\"diff_item_avg_spend_{j}w\"].fillna(\n            week_df[\"article_id\"].map(last_item_price)\n        )\n    \n        week_df[f\"item_avg_price_{j}w\"] = np.where(\n            week_df[f\"item_avg_price_{j}w\"].isna(),\n            week_df[f\"diff_item_avg_spend_{j}w\"] + week_df[f'avg_price_{j}w'],   # biểu thức trong cùng hàng\n            week_df[f\"item_avg_price_{j}w\"]\n        )\n\n        last_user_price = (\n            user_week_price[user_week_price[f\"diff_user_avg_spend_{j}w\"].notna()]\n            .sort_values(\"week\")\n            .groupby(\"customer_id\")[f\"diff_user_avg_spend_{j}w\"]\n            .first()\n        )\n    \n        week_df[f\"diff_user_avg_spend_{j}w\"] = week_df[f\"diff_user_avg_spend_{j}w\"].fillna(\n            week_df[\"customer_id\"].map(last_user_price)\n        )\n    \n        week_df[f\"user_avg_spend_{j}w\"] = np.where(\n            week_df[f\"user_avg_spend_{j}w\"].isna(),\n            week_df[f\"diff_user_avg_spend_{j}w\"] + week_df[f'avg_price_{j}w'],   # biểu thức trong cùng hàng\n            week_df[f\"user_avg_spend_{j}w\"]\n        )\n        del last_item_price, last_user_price\n        gc.collect()\n\n\n    item_last_buy_week = (\n        df_trans[df_trans[\"week\"] > 0]\n        .groupby(\"article_id\", as_index=False)[\"week\"]\n        .min()\n        .rename(columns={\"week\": \"item_last_buy_week\"})\n    )\n    item_last_buy_week['week_nums_last_item'] = item_last_buy_week['item_last_buy_week'] - 0\n\n    week_df = week_df.merge(\n        item_last_buy_week[[\"article_id\", \"week_nums_last_item\"]],\n        on=\"article_id\",\n        how=\"left\"\n    )\n    del item_last_buy_week, \n    week_df = week_df.drop(columns=[c for c in week_df.columns if c.startswith(\"diff\")])\n    week_df = week_df.drop(columns=['week'])\n    dataset = ds.dataset(os.path.join('/kaggle/input/dataset-process', f), format=\"parquet\")\n    writer = None\n\n    for batch in dataset.to_batches(batch_size=500_000):\n        base_chunk = batch.to_pandas()\n    \n        base_chunk = base_chunk.merge(\n            week_df,\n            on=[\"customer_id\", \"article_id\"],\n            how=\"left\"\n        )\n    \n        table = pa.Table.from_pandas(base_chunk)\n    \n        if writer is None:\n            writer = pq.ParquetWriter(\n                f,\n                table.schema\n            )\n    \n        writer.write_table(table)\n\n    writer.close()\n\n    gc.collect()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}