{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":15797858,"datasetId":9869115,"databundleVersionId":16744718}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2026-04-18T05:13:59.639973Z","iopub.execute_input":"2026-04-18T05:13:59.640621Z","iopub.status.idle":"2026-04-18T05:17:04.148561Z","shell.execute_reply.started":"2026-04-18T05:13:59.640591Z","shell.execute_reply":"2026-04-18T05:17:04.147714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install openai-clip --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:17:04.150452Z","iopub.execute_input":"2026-04-18T05:17:04.151247Z","iopub.status.idle":"2026-04-18T05:17:10.875590Z","shell.execute_reply.started":"2026-04-18T05:17:04.151217Z","shell.execute_reply":"2026-04-18T05:17:10.874550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sentence_transformers import SentenceTransformer\nimport numpy as np\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.decomposition import TruncatedSVD\nimport torch\nfrom clip import load\nmodel_clip, preprocess = load('ViT-B/32', device='cuda')\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport os\nfrom sklearn.preprocessing import normalize\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics import silhouette_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:17:10.877009Z","iopub.execute_input":"2026-04-18T05:17:10.877395Z","iopub.status.idle":"2026-04-18T05:17:44.869426Z","shell.execute_reply.started":"2026-04-18T05:17:10.877367Z","shell.execute_reply":"2026-04-18T05:17:44.868496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transaction = pd.read_parquet(\"/kaggle/input/datasets/huyenpham22/h-and-m-0-4-stratified-sample/train_val_data/hist_val.parquet\")\ntransaction['article_id'] = transaction['article_id'].astype(str)\n\ncust = pd.read_csv(\"/kaggle/input/datasets/huyenpham22/h-and-m-0-4-stratified-sample/customers.csv\")\n\narticles = pd.read_csv(\n    '/kaggle/input/datasets/huyenpham22/h-and-m-0-4-stratified-sample/articles.csv',\n    dtype={'article_id': str} \n)\n\n# Kiểm tra lại kiểu dữ liệu\nprint(f\"Transaction ID type: {type(transaction['article_id'].iloc[0])}\")\nprint(f\"Articles ID type: {type(articles['article_id'].iloc[0])}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:17:44.870531Z","iopub.execute_input":"2026-04-18T05:17:44.871160Z","iopub.status.idle":"2026-04-18T05:18:03.762211Z","shell.execute_reply.started":"2026-04-18T05:17:44.871133Z","shell.execute_reply":"2026-04-18T05:18:03.761554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DF để lưu\ndf_items = articles[['article_id']].copy()\ndf_users = cust[['customer_id']].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:18:03.763920Z","iopub.execute_input":"2026-04-18T05:18:03.764249Z","iopub.status.idle":"2026-04-18T05:18:03.825872Z","shell.execute_reply.started":"2026-04-18T05:18:03.764224Z","shell.execute_reply":"2026-04-18T05:18:03.825227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Embed item và user\n## Item","metadata":{}},{"cell_type":"code","source":"cat_cols = [\n    'product_group_name',\n    'colour_group_name', \n    'index_group_name',\n    'garment_group_name',\n    'section_name',\n    'index_name',\n]\n\nenc = OneHotEncoder(sparse_output=True, handle_unknown='ignore')\ncat_matrix = enc.fit_transform(articles[cat_cols].fillna('unknown'))\n\nsvd = TruncatedSVD(n_components=32, random_state=42)\ncat_vec = svd.fit_transform(cat_matrix)  # shape: (105542, 32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:18:03.826794Z","iopub.execute_input":"2026-04-18T05:18:03.827051Z","iopub.status.idle":"2026-04-18T05:18:04.949129Z","shell.execute_reply.started":"2026-04-18T05:18:03.827014Z","shell.execute_reply":"2026-04-18T05:18:04.948365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_text = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')\n\n# detail_desc có thể null → fillna\ntexts = articles['detail_desc'].fillna('').tolist()\n\n# Encode theo batch để không bị OOM\ntext_vec_full = model_text.encode(\n    texts,\n    batch_size=256,\n    show_progress_bar=True,\n    device='cuda'\n)  # shape: (105542, 384)\n\n# Giảm xuống 64d\nsvd_text = TruncatedSVD(n_components=64, random_state=42)\ntext_vec = svd_text.fit_transform(text_vec_full)  # shape: (105542, 64)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:18:04.949967Z","iopub.execute_input":"2026-04-18T05:18:04.950260Z","iopub.status.idle":"2026-04-18T05:19:37.866226Z","shell.execute_reply.started":"2026-04-18T05:18:04.950226Z","shell.execute_reply":"2026-04-18T05:19:37.865504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMAGE_DIR = '/kaggle/input/datasets/huyenpham22/h-and-m-0-4-stratified-sample/images'\n\nclass ArticleImageDataset(Dataset):\n    def __init__(self, article_ids, image_dir, preprocess):\n        self.article_ids = article_ids\n        self.image_dir   = image_dir\n        self.preprocess  = preprocess\n\n    def __len__(self):\n        return len(self.article_ids)\n\n    def __getitem__(self, idx):\n        aid    = self.article_ids[idx]\n        folder = str(aid)[:3]\n        path   = os.path.join(self.image_dir, f'0{folder}', f'0{aid}.jpg')\n        try:\n            img = self.preprocess(Image.open(path).convert('RGB'))\n        except:\n            img = torch.zeros(3, 224, 224)\n        return img, idx\n\narticle_ids = articles['article_id'].tolist()\ndataset     = ArticleImageDataset(article_ids, IMAGE_DIR, preprocess)\nloader      = DataLoader(dataset, batch_size=256, num_workers=2, pin_memory=True)\n\nimage_vecs = np.zeros((len(article_ids), 512))\n\nmodel_clip.eval()\nwith torch.no_grad():\n    for imgs, idxs in loader:\n        imgs  = imgs.to('cuda')\n        feats = model_clip.encode_image(imgs)\n        feats = feats.cpu().float().numpy()\n        image_vecs[idxs] = feats\n\nprint('Done:', image_vecs.shape)  # (105542, 512)\n\n# Giảm xuống 64d\nsvd_img   = TruncatedSVD(n_components=64, random_state=42)\nimage_vec = svd_img.fit_transform(image_vecs)\nprint('Image vec shape:', image_vec.shape)  # (105542, 64)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:19:37.867452Z","iopub.execute_input":"2026-04-18T05:19:37.867767Z","iopub.status.idle":"2026-04-18T05:21:07.858910Z","shell.execute_reply.started":"2026-04-18T05:19:37.867739Z","shell.execute_reply":"2026-04-18T05:21:07.858021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize từng phần trước khi concat\n# để không bị 1 nguồn dominate\ncat_vec_norm   = normalize(cat_vec)\ntext_vec_norm  = normalize(text_vec)\nimage_vec_norm = normalize(image_vec)\n\n# Concat\nitem_concat = np.hstack([\n    cat_vec_norm,    # 32d\n    text_vec_norm,   # 64d\n    image_vec_norm,  # 64d\n])  # shape: (105542, 160)\n\n# PCA xuống 128d\npca = PCA(n_components=128, random_state=42)\nitem_embed_vec = pca.fit_transform(item_concat)  # shape: (105542, 128)\nitem_embed_vec = normalize(item_embed_vec)        # normalize lần cuối\n\ndf_items['item_embed_vec'] = list(item_embed_vec)\n\ndf_items.to_parquet('df_items.parquet')\nprint('Saved df_items:', df_items.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:07.860380Z","iopub.execute_input":"2026-04-18T05:21:07.860765Z","iopub.status.idle":"2026-04-18T05:21:09.153553Z","shell.execute_reply.started":"2026-04-18T05:21:07.860734Z","shell.execute_reply":"2026-04-18T05:21:09.152934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_items.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:09.154598Z","iopub.execute_input":"2026-04-18T05:21:09.154929Z","iopub.status.idle":"2026-04-18T05:21:09.180392Z","shell.execute_reply.started":"2026-04-18T05:21:09.154891Z","shell.execute_reply":"2026-04-18T05:21:09.179749Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## User","metadata":{}},{"cell_type":"code","source":"transaction.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:09.181403Z","iopub.execute_input":"2026-04-18T05:21:09.181819Z","iopub.status.idle":"2026-04-18T05:21:09.187438Z","shell.execute_reply.started":"2026-04-18T05:21:09.181790Z","shell.execute_reply":"2026-04-18T05:21:09.186527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"col = transaction.columns\n\nfor c in col:\n    # .isna() creates a boolean mask, .sum() counts the True values\n    print(f\"{c}: {transaction[c].isna().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:09.188528Z","iopub.execute_input":"2026-04-18T05:21:09.188896Z","iopub.status.idle":"2026-04-18T05:21:12.671767Z","shell.execute_reply.started":"2026-04-18T05:21:09.188840Z","shell.execute_reply":"2026-04-18T05:21:12.671111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transaction.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:12.672661Z","iopub.execute_input":"2026-04-18T05:21:12.672934Z","iopub.status.idle":"2026-04-18T05:21:12.684574Z","shell.execute_reply.started":"2026-04-18T05:21:12.672898Z","shell.execute_reply":"2026-04-18T05:21:12.683787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transaction['t_dat'] = pd.to_datetime(transaction['t_dat'])\n\nREFERENCE_DATE = transaction['t_dat'].max()\nprint('Reference date:', REFERENCE_DATE)\n\nembed_map = {\n    aid: item_embed_vec[i]\n    for i, aid in enumerate(articles['article_id'].tolist())\n}\n\ntransaction['embed'] = transaction['article_id'].map(embed_map)\n\ntransaction['days_since'] = (REFERENCE_DATE - transaction['t_dat']).dt.days + 1\ntransaction['weight']     = 1 / transaction['days_since']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:12.688076Z","iopub.execute_input":"2026-04-18T05:21:12.688897Z","iopub.status.idle":"2026-04-18T05:21:18.384832Z","shell.execute_reply.started":"2026-04-18T05:21:12.688841Z","shell.execute_reply":"2026-04-18T05:21:18.383873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transaction.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:18.385874Z","iopub.execute_input":"2026-04-18T05:21:18.386231Z","iopub.status.idle":"2026-04-18T05:21:18.399974Z","shell.execute_reply.started":"2026-04-18T05:21:18.386204Z","shell.execute_reply":"2026-04-18T05:21:18.399367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bỏ SP không có embedding (rất hiếm)\ntransaction = transaction.dropna(subset=['embed'])\n\nprint('transaction sau dropna:', len(transaction))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:18.400848Z","iopub.execute_input":"2026-04-18T05:21:18.401218Z","iopub.status.idle":"2026-04-18T05:21:26.691292Z","shell.execute_reply.started":"2026-04-18T05:21:18.401177Z","shell.execute_reply":"2026-04-18T05:21:26.690429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_mean_embed(group):\n    embeds  = np.stack(group['embed'].values)  # (n, 128)\n    weights = group['weight'].values            # (n,)\n    weights = weights / weights.sum()           # normalize\n    return (embeds * weights[:, None]).sum(axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:26.692366Z","iopub.execute_input":"2026-04-18T05:21:26.692912Z","iopub.status.idle":"2026-04-18T05:21:26.696996Z","shell.execute_reply.started":"2026-04-18T05:21:26.692885Z","shell.execute_reply":"2026-04-18T05:21:26.696220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Đang tính user_vec...')\nuser_vec_all = (\n    transaction\n    .groupby('customer_id')\n    .apply(weighted_mean_embed)\n    .reset_index()\n    .rename(columns={0: 'user_vec'})\n)\nprint('user_vec done:', user_vec_all.shape)\n\n# user_vec_7d\nprint('Đang tính user_vec_7d...')\nmask_7d = transaction['t_dat'] >= REFERENCE_DATE - pd.Timedelta(days=7)\nuser_vec_7d = (\n    transaction[mask_7d]\n    .groupby('customer_id')\n    .apply(weighted_mean_embed)\n    .reset_index()\n    .rename(columns={0: 'user_vec_7d'})\n)\nprint('user_vec_7d done:', user_vec_7d.shape)\n\n# user_vec_14d\nprint('Đang tính user_vec_14d...')\nmask_14d = transaction['t_dat'] >= REFERENCE_DATE - pd.Timedelta(days=14)\nuser_vec_14d = (\n    transaction[mask_14d]\n    .groupby('customer_id')\n    .apply(weighted_mean_embed)\n    .reset_index()\n    .rename(columns={0: 'user_vec_14d'})\n)\nprint('user_vec_14d done:', user_vec_14d.shape)\n\n# user_vec_30d\nprint('Đang tính user_vec_30d...')\nmask_30d = transaction['t_dat'] >= REFERENCE_DATE - pd.Timedelta(days=30)\nuser_vec_30d = (\n    transaction[mask_30d]\n    .groupby('customer_id')\n    .apply(weighted_mean_embed)\n    .reset_index()\n    .rename(columns={0: 'user_vec_30d'})\n)\nprint('user_vec_30d done:', user_vec_30d.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:21:26.698015Z","iopub.execute_input":"2026-04-18T05:21:26.698386Z","iopub.status.idle":"2026-04-18T05:25:24.890765Z","shell.execute_reply.started":"2026-04-18T05:21:26.698363Z","shell.execute_reply":"2026-04-18T05:25:24.889834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_embed_df = (\n    user_vec_all\n    .merge(user_vec_7d,  on='customer_id', how='left')\n    .merge(user_vec_14d, on='customer_id', how='left')\n    .merge(user_vec_30d, on='customer_id', how='left')\n)\n\n# Fallback: KH không mua trong 7d/30d → dùng user_vec thay thế\ndef fill_fallback(row, col):\n    if row[col] is None or (isinstance(row[col], float) and np.isnan(row[col])):\n        return row['user_vec']\n    return row[col]\n\nuser_embed_df['user_vec_7d']  = user_embed_df.apply(\n    lambda r: fill_fallback(r, 'user_vec_7d'), axis=1)\nuser_embed_df['user_vec_14d']  = user_embed_df.apply(\n    lambda r: fill_fallback(r, 'user_vec_14d'), axis=1)\nuser_embed_df['user_vec_30d'] = user_embed_df.apply(\n    lambda r: fill_fallback(r, 'user_vec_30d'), axis=1)\n\nprint('User embed shape:', user_embed_df.shape)\nprint('Sample:')\nprint(user_embed_df.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:25:24.891882Z","iopub.execute_input":"2026-04-18T05:25:24.892232Z","iopub.status.idle":"2026-04-18T05:26:05.933916Z","shell.execute_reply.started":"2026-04-18T05:25:24.892195Z","shell.execute_reply":"2026-04-18T05:26:05.933137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize từng vector\ndef normalize_vec_col(df, col):\n    mat = np.stack(df[col].values)\n    mat = normalize(mat)\n    df[col] = list(mat)\n    return df\n\nuser_embed_df = normalize_vec_col(user_embed_df, 'user_vec')\nuser_embed_df = normalize_vec_col(user_embed_df, 'user_vec_7d')\nuser_embed_df = normalize_vec_col(user_embed_df, 'user_vec_14d')\nuser_embed_df = normalize_vec_col(user_embed_df, 'user_vec_30d')\n\ndf_users = df_users.merge(\n    user_embed_df[['customer_id', 'user_vec', \n                   'user_vec_7d', 'user_vec_14d', 'user_vec_30d']],\n    on='customer_id',\n    how='left'\n)\ndf_users.to_parquet('df_users.parquet')\nprint('Saved df_users:', df_users.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:26:05.934878Z","iopub.execute_input":"2026-04-18T05:26:05.935162Z","iopub.status.idle":"2026-04-18T05:27:02.182639Z","shell.execute_reply.started":"2026-04-18T05:26:05.935137Z","shell.execute_reply":"2026-04-18T05:27:02.181898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_embed_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:27:02.184792Z","iopub.execute_input":"2026-04-18T05:27:02.185296Z","iopub.status.idle":"2026-04-18T05:27:02.192309Z","shell.execute_reply.started":"2026-04-18T05:27:02.185253Z","shell.execute_reply":"2026-04-18T05:27:02.191106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_embed_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:27:02.193497Z","iopub.execute_input":"2026-04-18T05:27:02.193985Z","iopub.status.idle":"2026-04-18T05:27:07.341748Z","shell.execute_reply.started":"2026-04-18T05:27:02.193939Z","shell.execute_reply":"2026-04-18T05:27:07.341083Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cluster","metadata":{}},{"cell_type":"code","source":"print('=== Item Clustering ===')\nk_values_item = [30, 50, 75, 100, 150]\nitem_scores   = []\nitem_inertias = []\n\nfor k in k_values_item:\n    km     = KMeans(n_clusters=k, random_state=42, n_init=10)\n    labels = km.fit_predict(item_embed_vec)\n    score  = silhouette_score(item_embed_vec, labels, sample_size=10000, random_state=42)\n    item_scores.append(score)\n    item_inertias.append(km.inertia_)\n    print(f'k={k:4d} | silhouette={score:.4f} | inertia={km.inertia_:.0f}')\n\n# Plot item\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].plot(k_values_item, item_scores, marker='o')\naxes[0].set_title('Item — Silhouette Score')\naxes[0].set_xlabel('k')\naxes[0].set_ylabel('Score')\naxes[1].plot(k_values_item, item_inertias, marker='o')\naxes[1].set_title('Item — Elbow')\naxes[1].set_xlabel('k')\naxes[1].set_ylabel('Inertia')\nplt.tight_layout()\nplt.show()\n\n# Chọn k tốt nhất\nbest_k_item = k_values_item[np.argmax(item_scores)]\nprint(f'\\nBest k item: {best_k_item}')\n\n# Train lại với best k\nprint(f'Training item KMeans k={best_k_item}...')\nkmeans_item         = KMeans(n_clusters=best_k_item, random_state=42, n_init=10)\nitem_cluster_labels = kmeans_item.fit_predict(item_embed_vec)\ndf_items['item_cluster_id'] = item_cluster_labels\n\nprint('Phân bổ item cluster:')\nitem_sizes = pd.Series(item_cluster_labels).value_counts()\nprint(item_sizes.describe())\nprint(f'Cluster < 50 SP: {(item_sizes < 50).sum()} cluster')\n\n# Lưu centroids\nitem_cluster_centroids = kmeans_item.cluster_centers_\nnp.save('item_cluster_centroids.npy', item_cluster_centroids)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:27:07.342839Z","iopub.execute_input":"2026-04-18T05:27:07.343303Z","iopub.status.idle":"2026-04-18T05:29:44.886441Z","shell.execute_reply.started":"2026-04-18T05:27:07.343275Z","shell.execute_reply":"2026-04-18T05:29:44.885548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kiểm tra shape của từng vector\nshapes = df_users['user_vec'].apply(lambda x: np.array(x).shape if x is not None else 'None')\nprint(shapes.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:29:44.887704Z","iopub.execute_input":"2026-04-18T05:29:44.888114Z","iopub.status.idle":"2026-04-18T05:29:45.999211Z","shell.execute_reply.started":"2026-04-18T05:29:44.888075Z","shell.execute_reply":"2026-04-18T05:29:45.998355Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Bỏ customer chưa có mua gì","metadata":{}},{"cell_type":"code","source":"zero_vec = np.zeros(128)\n\n# Tách KH có transaction và không có transaction\nmask_has_purchase = df_users['user_vec'].apply(\n    lambda x: np.array(x).shape == (128,)\n)\n\ndf_users_purchase   = df_users[mask_has_purchase].copy()   # KH có lịch sử mua\ndf_users_coldstart = df_users[~mask_has_purchase].copy() # KH chưa mua\n\nprint(f'KH có lịch sử mua:  {len(df_users_purchase):,}')\nprint(f'KH cold-start:      {len(df_users_coldstart):,}')\n\n# Chỉ cluster KH active\nuser_vec_matrix = np.stack(df_users_purchase['user_vec'].values)\nprint('user_vec_matrix shape:', user_vec_matrix.shape)  # (1291147, 128)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:29:46.000344Z","iopub.execute_input":"2026-04-18T05:29:46.001186Z","iopub.status.idle":"2026-04-18T05:29:49.186194Z","shell.execute_reply.started":"2026-04-18T05:29:46.001157Z","shell.execute_reply":"2026-04-18T05:29:49.185376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.cluster import MiniBatchKMeans\n\n# ── User clustering ──────────────────────────────────────\nprint('\\n=== User Clustering ===')\nuser_vec_matrix = np.stack(df_users_purchase['user_vec'].values)\nk_values_user   = [20, 30, 50, 75, 100]\nuser_scores     = []\nuser_inertias   = []\n\nfor k in k_values_user:\n    km     = MiniBatchKMeans(n_clusters=k, random_state=42, n_init=10, batch_size=10000)\n    labels = km.fit_predict(user_vec_matrix)\n    score  = silhouette_score(user_vec_matrix, labels, sample_size=10000, random_state=42)\n    user_scores.append(score)\n    user_inertias.append(km.inertia_)\n    print(f'k={k:4d} | silhouette={score:.4f} | inertia={km.inertia_:.0f}')\n\n# Plot\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].plot(k_values_user, user_scores, marker='o')\naxes[0].set_title('User — Silhouette Score')\naxes[0].set_xlabel('k')\naxes[0].set_ylabel('Score')\naxes[1].plot(k_values_user, user_inertias, marker='o')\naxes[1].set_title('User — Elbow')\naxes[1].set_xlabel('k')\naxes[1].set_ylabel('Inertia')\nplt.tight_layout()\nplt.show()\n\n# Chọn k tốt nhất\nbest_k_user = k_values_user[np.argmax(user_scores)]\nprint(f'\\nBest k user (theo silhouette): {best_k_user}')\n\n# Kiểm tra min size\nkm_check     = MiniBatchKMeans(n_clusters=best_k_user, random_state=42, n_init=10, batch_size=10000)\nlabels_check = km_check.fit_predict(user_vec_matrix)\nsize_check   = pd.Series(labels_check).value_counts()\nmin_size     = size_check.min()\n\nif min_size < 500:\n    print(f'Cảnh báo: cluster nhỏ nhất chỉ có {min_size} KH')\n    for k, score in zip(k_values_user, user_scores):\n        km_tmp     = MiniBatchKMeans(n_clusters=k, random_state=42, n_init=10, batch_size=10000)\n        labels_tmp = km_tmp.fit_predict(user_vec_matrix)\n        min_s      = pd.Series(labels_tmp).value_counts().min()\n        if min_s >= 500:\n            best_k_user = k\n            print(f'Chọn k={k} (silhouette={score:.4f}, min_size={min_s})')\n            break\n\n# Train lại với best k\nprint(f'\\nTraining MiniBatchKMeans k={best_k_user}...')\nkmeans_user         = MiniBatchKMeans(n_clusters=best_k_user, random_state=42, n_init=10, batch_size=10000)\nuser_cluster_labels = kmeans_user.fit_predict(user_vec_matrix)\n\n# Gắn vào active users\ndf_users_purchase['customer_cluster_id'] = user_cluster_labels\n\n# Cold-start → -1\ndf_users_coldstart['customer_cluster_id'] = -1\n\n# Merge lại\ndf_users = pd.concat([df_users_purchase, df_users_coldstart], ignore_index=True)\n\nprint('Phân bổ user cluster:')\nuser_sizes = pd.Series(user_cluster_labels).value_counts()\nprint(user_sizes.describe())\nprint(f'Cluster < 500 KH: {(user_sizes < 500).sum()} cluster')\n\n# Lưu centroids\nuser_cluster_centroids = kmeans_user.cluster_centers_\nnp.save('user_cluster_centroids.npy', user_cluster_centroids)\n\n# Save\ndf_users.to_parquet('df_users.parquet')\nprint('Saved df_users:', df_users.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:29:49.187226Z","iopub.execute_input":"2026-04-18T05:29:49.187644Z","iopub.status.idle":"2026-04-18T05:31:13.714140Z","shell.execute_reply.started":"2026-04-18T05:29:49.187616Z","shell.execute_reply":"2026-04-18T05:31:13.713228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Save ─────────────────────────────────────────────────\ndf_items.to_parquet('df_items.parquet')\ndf_users.to_parquet('df_users.parquet')\nprint('\\nSaved df_items:', df_items.shape)\nprint('Saved df_users:', df_users.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:31:13.718382Z","iopub.execute_input":"2026-04-18T05:31:13.718971Z","iopub.status.idle":"2026-04-18T05:31:56.369816Z","shell.execute_reply.started":"2026-04-18T05:31:13.718942Z","shell.execute_reply":"2026-04-18T05:31:56.369110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport random\nimport os\n\ndef get_items_in_cluster(cluster_id, n=10):\n    ids = df_items[df_items['item_cluster_id'] == cluster_id]['article_id'].tolist()\n    print(f'Cluster {cluster_id}: {len(ids)} SP')\n    sample = random.sample(ids, min(n, len(ids)))\n    return sample\n\ndef show_cluster_images(cluster_id, n=10, image_dir=IMAGE_DIR):\n    article_ids = get_items_in_cluster(cluster_id, n)\n    \n    cols = 5\n    rows = (len(article_ids) + cols - 1) // cols\n    fig, axes = plt.subplots(rows, cols, figsize=(cols * 3, rows * 3))\n    axes = axes.flatten()\n\n    for i, aid in enumerate(article_ids):\n        aid_str = str(int(aid)).zfill(10)\n        folder  = aid_str[:3]\n        path    = os.path.join(image_dir, folder, f'{aid_str}.jpg')\n\n        ax = axes[i]\n        if os.path.exists(path):\n            img = mpimg.imread(path)\n            ax.imshow(img)\n        else:\n            info = articles[articles['article_id'] == aid]\n            if len(info) > 0:\n                name   = info['prod_name'].values[0]\n                colour = info['colour_group_name'].values[0]\n                group  = info['product_group_name'].values[0]\n                label  = f'{name}\\n{colour}\\n{group}'\n            else:\n                label = 'No info'\n            ax.text(0.5, 0.5, label,\n                    ha='center', va='center',\n                    transform=ax.transAxes,\n                    fontsize=7)\n            ax.set_facecolor('#f0f0f0')\n\n        ax.set_title(f'{aid_str}', fontsize=7)\n        ax.axis('off')\n\n    for j in range(len(article_ids), len(axes)):\n        axes[j].axis('off')\n\n    plt.suptitle(f'Cluster {cluster_id} — {len(article_ids)} SP', fontsize=12)\n    plt.tight_layout()\n    plt.show()\n\nshow_cluster_images(cluster_id=10, n=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:31:56.372099Z","iopub.execute_input":"2026-04-18T05:31:56.372478Z","iopub.status.idle":"2026-04-18T05:32:06.395746Z","shell.execute_reply.started":"2026-04-18T05:31:56.372449Z","shell.execute_reply":"2026-04-18T05:32:06.393589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bổ sung cái feature vốn có","metadata":{}},{"cell_type":"code","source":"# Drop các cột demographic cũ trước\ncols_to_drop = [c for c in df_users.columns if c.endswith('_x') or c.endswith('_y')\n                or c in ['FN', 'Active', 'club_member_status', \n                         'fashion_news_frequency', 'age', 'age_binning']]\ndf_users = df_users.drop(columns=cols_to_drop, errors='ignore')\n\nprint('Sau khi drop:', df_users.columns.tolist())\n\n# Merge lại 1 lần\ndf_users = df_users.merge(\n    cust[['customer_id', 'FN', 'Active', 'club_member_status',\n          'fashion_news_frequency', 'age']],\n    on='customer_id',\n    how='left'\n)\n\n# FN và Active\ndf_users['FN']     = df_users['FN'].fillna(0).astype(int)\ndf_users['Active'] = df_users['Active'].fillna(0).astype(int)\n\n# club_member_status\nclub_map = {'ACTIVE': 2, 'PRE-CREATE': 1, 'LEFT CLUB': -1}\ndf_users['club_member_status'] = df_users['club_member_status'].map(club_map).fillna(0).astype(int)\n\n# fashion_news_frequency\nnews_map = {'Regularly': 2, 'Monthly': 1, 'NONE': 0, 'None': 0}\ndf_users['fashion_news_frequency'] = df_users['fashion_news_frequency'].map(news_map).fillna(0).astype(int)\n\n# Age\ndf_users['age'] = df_users['age'].fillna(df_users['age'].median()).astype('int8')\n\n# Age Binning\nbins   = [0, 25, 35, 45, 55, 65, 999]\nlabels = [0,  1,  2,  3,  4,  5]\ndf_users['age_binning'] = pd.cut(\n    df_users['age'], bins=bins, labels=labels, right=True\n).astype('int8')\n\nprint(df_users[['customer_id', 'FN', 'Active', 'club_member_status',\n                'fashion_news_frequency', 'age', 'age_binning']].head(5))\nprint('\\nShape:', df_users.shape)\n\ndf_users.to_parquet('df_users.parquet')\nprint('Saved')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:32:06.396811Z","iopub.execute_input":"2026-04-18T05:32:06.397251Z","iopub.status.idle":"2026-04-18T05:32:52.667012Z","shell.execute_reply.started":"2026-04-18T05:32:06.397207Z","shell.execute_reply":"2026-04-18T05:32:52.666077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ── Lấy từ data gốc ─────────────────────────────────────\ndf_items = df_items.merge(\n    articles[['article_id', 'index_group_no', \n              'colour_group_name', 'graphical_appearance_name']],\n    on='article_id',\n    how='left'\n)\n\n# ── Encode index_group_no ────────────────────────────────\n# Đã là số rồi → giữ nguyên, chỉ fillna\ndf_items['index_group_no'] = df_items['index_group_no'].fillna(-1).astype(int)\n\n# ── Encode colour_group ──────────────────────────────────\nfrom sklearn.preprocessing import LabelEncoder\n\nle_colour = LabelEncoder()\ndf_items['colour_group'] = le_colour.fit_transform(\n    df_items['colour_group_name'].fillna('Unknown')\n)\n\n# ── Encode graphical_appearance ──────────────────────────\nle_graph = LabelEncoder()\ndf_items['graphical_appearance'] = le_graph.fit_transform(\n    df_items['graphical_appearance_name'].fillna('Unknown')\n)\n\n# ── Kiểm tra ─────────────────────────────────────────────\nprint(df_items[['article_id', 'index_group_no', \n                'colour_group', 'graphical_appearance']].head(5))\nprint('\\nShape:', df_items.shape)\n\n# ── Save ─────────────────────────────────────────────────\ndf_items.to_parquet('df_items.parquet')\nprint('Saved df_items:', df_items.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:32:52.669197Z","iopub.execute_input":"2026-04-18T05:32:52.669551Z","iopub.status.idle":"2026-04-18T05:32:53.583669Z","shell.execute_reply.started":"2026-04-18T05:32:52.669525Z","shell.execute_reply":"2026-04-18T05:32:53.582830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_items.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:32:53.584784Z","iopub.execute_input":"2026-04-18T05:32:53.585140Z","iopub.status.idle":"2026-04-18T05:32:56.295000Z","shell.execute_reply.started":"2026-04-18T05:32:53.585106Z","shell.execute_reply":"2026-04-18T05:32:56.294123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_users.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-18T05:32:56.296798Z","iopub.execute_input":"2026-04-18T05:32:56.297316Z","iopub.status.idle":"2026-04-18T05:32:56.325705Z","shell.execute_reply.started":"2026-04-18T05:32:56.297288Z","shell.execute_reply":"2026-04-18T05:32:56.324838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"scrolled":true},"outputs":[],"execution_count":null}]}