{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":1864562,"sourceType":"datasetVersion","datasetId":1109702},{"sourceId":12608713,"sourceType":"datasetVersion","datasetId":7964561},{"sourceId":12683273,"sourceType":"datasetVersion","datasetId":8015397},{"sourceId":12683682,"sourceType":"datasetVersion","datasetId":8015669},{"sourceId":12697009,"sourceType":"datasetVersion","datasetId":8024161},{"sourceId":12697321,"sourceType":"datasetVersion","datasetId":8024355},{"sourceId":12753492,"sourceType":"datasetVersion","datasetId":8062257},{"sourceId":12755574,"sourceType":"datasetVersion","datasetId":8063653},{"sourceId":12764062,"sourceType":"datasetVersion","datasetId":8068977},{"sourceId":12764725,"sourceType":"datasetVersion","datasetId":8069396}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**EDA**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Đọc dữ liệu customers từ bộ dataset H&M\n# Đường dẫn này đúng nếu bạn đang ở môi trường Kaggle\ncustomers = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\")\n\n# Nhóm theo tuổi và đếm số lượng khách hàng\ntemp = customers.groupby([\"age\"])[\"customer_id\"].count()\ndf = pd.DataFrame({\"Age\": temp.index, \"Customers\": temp.values})\ndf = df.sort_values([\"Age\"], ascending=False)\n\n# Vẽ biểu đồ với màu magma\nplt.figure(figsize=(20, 10))\nplt.title(\"NUMBER OF CUSTOMERS BY AGE\")\ns = sns.barplot(x=\"Age\", y=\"Customers\", data=df, palette=\"rocket\")\ns.set_xticklabels(s.get_xticklabels(), rotation=90)\nplt.savefig(\"/kaggle/working/customers_by_age.png\", dpi=300, bbox_inches='tight')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T06:27:18.146603Z","iopub.execute_input":"2025-08-12T06:27:18.147371Z","iopub.status.idle":"2025-08-12T06:27:24.627388Z","shell.execute_reply.started":"2025-08-12T06:27:18.147337Z","shell.execute_reply":"2025-08-12T06:27:24.626104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Read dataset and parse dates\ntrain = pd.read_csv(\n    \"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\",\n    parse_dates=['t_dat']\n)\n\n# Plot daily article sales\nplt.figure(figsize=(16, 9))\ntrain.groupby('t_dat')['article_id'].count().plot(color='orange', linewidth=2)\n\n# Add labels and title\nplt.title(\"Number of Products Sold per Day\", fontsize=16)\nplt.xlabel(\"Date\", fontsize=12)\nplt.ylabel(\"Number of Products Sold\", fontsize=12)\n\n# Save to Kaggle output folder\nplt.savefig(\"/kaggle/working/daily_sales.png\", dpi=300, bbox_inches='tight')\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T06:32:40.859116Z","iopub.execute_input":"2025-08-12T06:32:40.859513Z","iopub.status.idle":"2025-08-12T06:33:28.991051Z","shell.execute_reply.started":"2025-08-12T06:32:40.859485Z","shell.execute_reply":"2025-08-12T06:33:28.989826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input/h-and-m-personalized-fashion-recommendations\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T06:56:20.166714Z","iopub.execute_input":"2025-08-12T06:56:20.167186Z","iopub.status.idle":"2025-08-12T06:56:20.174278Z","shell.execute_reply.started":"2025-08-12T06:56:20.167158Z","shell.execute_reply":"2025-08-12T06:56:20.173162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.image import imread\n\n# Hàm xác định mùa\ndef get_season(date):\n    month = date.month\n    if month in [12, 1, 2]:\n        return 'Summer'\n    elif month in [3, 4, 5]:\n        return 'Autumn'\n    elif month in [6, 7, 8]:\n        return 'Winter'\n    else:\n        return 'Spring'\n\n# Hàm lấy đường dẫn ảnh\ndef image_lookup_path(g_id):\n    g_id_str = str(g_id).zfill(10)  # đủ 10 ký tự\n    return f\"/kaggle/input/h-and-m-personalized-fashion-recommendations/images/{g_id_str[:3]}/{g_id_str}.jpg\"\n\n\n# Tạo cột season\ntrain['season'] = train['t_dat'].apply(get_season)\n\n# Lấy top 12 sản phẩm bán chạy trong mùa Summer\nsummer_top = train[train['season'] == 'Summer']['article_id'].value_counts().head(12).index\n\n# Vẽ ảnh\nfig, ax = plt.subplots(3, 4, figsize=(15, 10))\nax = ax.flatten()\nfig.suptitle(\"Top 12 Best-Selling Products - Summer\", fontsize=22, color='red')\n\nfor i, art_id in enumerate(summer_top):\n    try:\n        img = imread(image_lookup_path(art_id))\n        ax[i].imshow(img)\n        ax[i].set_title(str(art_id), fontsize=10)\n        ax[i].axis('off')\n    except FileNotFoundError:\n        ax[i].text(0.5, 0.5, 'Image Not Found', fontsize=8, ha='center', va='center')\n        ax[i].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T07:06:46.649241Z","iopub.execute_input":"2025-08-12T07:06:46.649628Z","iopub.status.idle":"2025-08-12T07:07:56.596366Z","shell.execute_reply.started":"2025-08-12T07:06:46.649600Z","shell.execute_reply":"2025-08-12T07:07:56.595133Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Bước 1: Hiểu và đọc dữ liệu (Data Understanding)**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n#Tham chiếu tới dữ liệu \ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\n#Bắt đầu xử lý dữ liệu\nprint(transactions.head())\nprint(customers.head())\nprint(articles.head())\n#Kiểm tra dữ liệu bị thiếu \nprint(\"Transactions missing values:\\n\", transactions.isnull().sum())\nprint(\"Customers missing values:\\n\", customers.isnull().sum())\nprint(\"Articles missing values:\\n\", articles.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:24:52.883221Z","iopub.execute_input":"2025-07-29T08:24:52.884382Z","iopub.status.idle":"2025-07-29T08:25:54.884522Z","shell.execute_reply.started":"2025-07-29T08:24:52.884342Z","shell.execute_reply":"2025-07-29T08:25:54.883570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tuổi khách hàng\nprint(\"📊 Tuổi khách hàng:\")\nprint(customers['age'].describe())\n\n# Giá sản phẩm\nprint(\"\\n📊 Giá sản phẩm đã mua:\")\nprint(transactions['price'].describe())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:25:54.886219Z","iopub.execute_input":"2025-07-29T08:25:54.886489Z","iopub.status.idle":"2025-07-29T08:25:56.286946Z","shell.execute_reply.started":"2025-07-29T08:25:54.886466Z","shell.execute_reply":"2025-07-29T08:25:56.286034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"👤 Số khách hàng:\", customers['customer_id'].nunique())\nprint(\"🛍️ Sản phẩm duy nhất:\", articles['article_id'].nunique())\nprint(\"📄 Tổng số giao dịch:\", transactions.shape[0])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:25:56.287859Z","iopub.execute_input":"2025-07-29T08:25:56.288081Z","iopub.status.idle":"2025-07-29T08:25:56.892987Z","shell.execute_reply.started":"2025-07-29T08:25:56.288063Z","shell.execute_reply":"2025-07-29T08:25:56.892077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Nếu cột season_code hoặc season_name có trong articles\nif 'season_code' in articles.columns:\n    print(\"Phân bố theo mùa:\")\n    print(articles['season_code'].value_counts())\n\nprint(\"\\nPhân bố loại sản phẩm:\")\nprint(articles['product_type_name'].value_counts().head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:25:56.893917Z","iopub.execute_input":"2025-07-29T08:25:56.894190Z","iopub.status.idle":"2025-07-29T08:25:56.909474Z","shell.execute_reply.started":"2025-07-29T08:25:56.894171Z","shell.execute_reply":"2025-07-29T08:25:56.908535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Chuyển article_id sang string\narticles['article_id'] = articles['article_id'].astype(str)\n# Xử lý missing: cột duy nhất thiếu là 'detail_desc' → thay NaN bằng chuỗi rỗng ''\nif 'detail_desc' in articles.columns:\n    articles['detail_desc'] = articles['detail_desc'].fillna('')\n# Xử lý trùng lặp theo article_id\narticles.drop_duplicates(subset='article_id', inplace=True)\n# Reset index\narticles.reset_index(drop=True, inplace=True)\n# Xem kết quả\narticles.info()\n\n# Chuyển các ID sang kiểu chuỗi để dễ xử lý\ntransactions['customer_id'] = transactions['customer_id'].astype(str)\ntransactions['article_id'] = transactions['article_id'].astype(str)\n# Chuyển t_dat về kiểu datetime\nif transactions['t_dat'].dtype == 'object':\n    transactions['t_dat'] = pd.to_datetime(transactions['t_dat'], errors='coerce')\n# Xử lý giá trị price không hợp lệ: giá <= 0 hoặc quá cao (ví dụ > 1)\ntransactions = transactions[(transactions['price'] > 0) & (transactions['price'] < 1)]\n# Loại bỏ trùng lặp tuyệt đối (nếu có)\ntransactions.drop_duplicates(inplace=True)\n# Reset index nếu cần\ntransactions.reset_index(drop=True, inplace=True)\n# Xem kết quả\ntransactions.info()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:25:56.911878Z","iopub.execute_input":"2025-07-29T08:25:56.912213Z","iopub.status.idle":"2025-07-29T08:26:46.327318Z","shell.execute_reply.started":"2025-07-29T08:25:56.912173Z","shell.execute_reply":"2025-07-29T08:26:46.326414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tạo cột month từ t_dat\ntransactions['month'] = transactions['t_dat'].dt.month\ntransactions['t_dat'] = pd.to_datetime(transactions['t_dat'], errors='coerce')\n# Gán mùa dựa vào tháng cửa hàng thụy điển nên lấy lịch nước ngoài \ndef assign_season(month):\n    if month in [12, 1, 2]:\n        return 'Winter'\n    elif month in [3, 4, 5]:\n        return 'Spring'\n    elif month in [6, 7, 8]:\n        return 'Summer'\n    else:\n        return 'Fall'\ntransactions['season'] = transactions['month'].apply(assign_season)\n# Hàm phân loại nhóm tuổi\ndef age_group(age):\n    if 1< age < 12:\n        return 'Childhood'\n    elif 12 <= age < 18:\n        return 'Adolescent'\n    elif 18< age < 45:\n        return 'Adult'\n    else:\n        return 'Senior'\ncustomers['age_group'] = customers['age'].apply(age_group)\ncustomers['gender'] = 'Unknown'\n# Nối transactions với customers\ndata = transactions.merge(customers, on='customer_id', how='inner')\n# Nối thêm articles để lấy thông tin sản phẩm\ndata = data.merge(articles[['article_id', 'product_type_name', 'colour_group_name', 'garment_group_name']], on='article_id', how='inner')\n# Giữ lại các cột cần thiết\nfinal_data = data[[\n    'customer_id',\n    'article_id',\n    'season',\n    'age_group',\n    'gender',\n    'product_type_name',\n    'colour_group_name',\n    'garment_group_name',\n    'price',\n    't_dat'\n]]\n# Xem kết quả\nfinal_data.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:26:46.328179Z","iopub.execute_input":"2025-07-29T08:26:46.328431Z","iopub.status.idle":"2025-07-29T08:27:57.801695Z","shell.execute_reply.started":"2025-07-29T08:26:46.328411Z","shell.execute_reply":"2025-07-29T08:27:57.800797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hàm tự động giảm bộ nhớ\ndef reduce_memory_usage(df):\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n    print(f\"Memory usage before: {start_mem:.2f} MB\")\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type == object:\n            num_unique = df[col].nunique()\n            num_total = len(df[col])\n            if num_unique / num_total < 0.5:\n                df[col] = df[col].astype('category')\n        elif pd.api.types.is_numeric_dtype(col_type):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if pd.api.types.is_integer_dtype(col_type):\n                if c_min >= 0:\n                    if c_max < 255:\n                        df[col] = df[col].astype('uint8')\n                    elif c_max < 65535:\n                        df[col] = df[col].astype('uint16')\n                    elif c_max < 2**31:\n                        df[col] = df[col].astype('uint32')\n                else:\n                    df[col] = df[col].astype('int32')\n            else:\n                df[col] = df[col].astype('float32')\n\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n    print(f\"Memory usage after: {end_mem:.2f} MB (↓ {100*(start_mem - end_mem)/start_mem:.1f}%)\")\n    return df\ntransactions = reduce_memory_usage(transactions)\ncustomers = reduce_memory_usage(customers)\narticles = reduce_memory_usage(articles)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:27:57.802713Z","iopub.execute_input":"2025-07-29T08:27:57.803139Z","iopub.status.idle":"2025-07-29T08:28:53.375037Z","shell.execute_reply.started":"2025-07-29T08:27:57.803110Z","shell.execute_reply":"2025-07-29T08:28:53.374127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tạo bản sao an toàn từ data\nfinal_data = data[[\n    'customer_id',\n    'article_id',\n    'season',\n    'age_group',\n    'gender',\n    'product_type_name',\n    'colour_group_name',\n    'garment_group_name',\n    'price',\n    't_dat'\n]].copy()\n\n# Trích xuất đặc trưng\nfinal_data['purchase_count'] = final_data.groupby('customer_id')['article_id'].transform('count')\nfinal_data['total_spent'] = final_data.groupby('customer_id')['price'].transform('sum')\nfinal_data['article_popularity'] = final_data.groupby('article_id')['customer_id'].transform('count')\nfinal_data['avg_customer_price'] = final_data.groupby('customer_id')['price'].transform('mean')\nfinal_data['season_agegroup_freq'] = final_data.groupby(['season', 'age_group'])['article_id'].transform('count')\nfinal_data['customer_garment_group_freq'] = final_data.groupby(['customer_id', 'garment_group_name'])['article_id'].transform('count')\nfinal_data['product_type_popularity'] = final_data.groupby('product_type_name')['customer_id'].transform('count')\n\n# Tính số ngày kể từ lần mua cuối cùng (Recency)\nlatest_date = final_data['t_dat'].max()\nfinal_data['recency_days'] = (latest_date - final_data['t_dat']).dt.days","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:28:53.376074Z","iopub.execute_input":"2025-07-29T08:28:53.376435Z","iopub.status.idle":"2025-07-29T08:30:32.523887Z","shell.execute_reply.started":"2025-07-29T08:28:53.376406Z","shell.execute_reply":"2025-07-29T08:30:32.523083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Bản sao để tránh cảnh báo SettingWithCopyWarning\nencoded_data = final_data.copy()\n\n# Xác định các cột phân loại cần mã hóa\ncat_cols = ['season', 'age_group', 'gender', \n            'product_type_name', 'colour_group_name', 'garment_group_name']\n\n# Dictionary để lưu encoder từng cột nếu muốn inverse_transform sau này\nle_dict = {}\n\n# Mã hóa từng cột bằng LabelEncoder\nfor col in cat_cols:\n    le = LabelEncoder()\n    encoded_data[col] = le.fit_transform(encoded_data[col])\n    le_dict[col] = le\n\n# Kiểm tra kết quả\nprint(encoded_data[cat_cols].head())\nprint(encoded_data.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:30:32.524784Z","iopub.execute_input":"2025-07-29T08:30:32.525112Z","iopub.status.idle":"2025-07-29T08:30:38.680783Z","shell.execute_reply.started":"2025-07-29T08:30:32.525088Z","shell.execute_reply":"2025-07-29T08:30:38.679500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(le_dict['season'].classes_)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:30:38.681460Z","iopub.status.idle":"2025-07-29T08:30:38.681827Z","shell.execute_reply.started":"2025-07-29T08:30:38.681635Z","shell.execute_reply":"2025-07-29T08:30:38.681652Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **BẮT ĐẦU CHẠY TỪ ĐÂY, chạy từ trên coi nó nặng qá**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndef split_for_train(base_path: str):\n    transactions = pd.read_csv(f'{base_path}/transactions_train.csv', parse_dates=['t_dat'])\n    customers = pd.read_csv(f'{base_path}/customers.csv')\n    articles = pd.read_csv(f'{base_path}/articles.csv')\n    start_date = transactions['t_dat'].min()\n    end_date = transactions['t_dat'].max()\n    split_date = start_date + 0.8 * (end_date - start_date)\n    \n    print(f\"Ngày chia 80/20: {split_date.date()}\")\n    train_df = transactions[transactions['t_dat'] < split_date]\n    test_df = transactions[transactions['t_dat'] >= split_date]\n\n    print(f\"Train size: {len(train_df)}, Test size: {len(test_df)}\")\n\n    # 4. Lọc customers và articles theo train\n    train_customers = train_df['customer_id'].unique()\n    train_articles = train_df['article_id'].unique()\n\n    filtered_customers_train = customers[customers['customer_id'].isin(train_customers)]\n    filtered_articles_train = articles[articles['article_id'].isin(train_articles)]\n\n    return train_df, test_df, filtered_customers_train, filtered_articles_train\n\n\n# CÓ CẦN LƯU FILE CSVKK\n# train_df.to_csv(\"transactions_train_80.csv\", index=False)\n# test_df.to_csv(\"transactions_test_20.csv\", index=False)\n# Gọi hàm với thư mục chứa dữ liệu (từ Kaggle)\ntrain_df, test_df, customers_train, articles_train = split_for_train('/kaggle/input/h-and-m-personalized-fashion-recommendations')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T02:55:40.151134Z","iopub.execute_input":"2025-08-13T02:55:40.151367Z","iopub.status.idle":"2025-08-13T02:57:27.817876Z","shell.execute_reply.started":"2025-08-13T02:55:40.151346Z","shell.execute_reply":"2025-08-13T02:57:27.816318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_label_counts(train_df, test_df):\n    # Count the number of unique article IDs in each dataset\n    original_labels = len(set(train_df['article_id']).union(set(test_df['article_id'])))\n    train_labels = len(set(train_df['article_id']))\n    test_labels = len(set(test_df['article_id']))\n\n    # Data for plotting\n    categories = ['Original Ratings', 'Train Set', 'Test Set']\n    counts = [original_labels, train_labels, test_labels]\n\n    # Creating the bar plot\n    plt.figure(figsize=(10, 6))\n    plt.bar(categories, counts, color=['blue', 'orange', 'green'])\n    plt.title('Number of Unique Article IDs in Datasets', fontsize=16)\n    plt.xlabel('Dataset', fontsize=14)\n    plt.ylabel('Number of Unique Article IDs', fontsize=14)\n    plt.xticks(rotation=15)\n    \n    # Adding data labels on top of the bars\n    for i, count in enumerate(counts):\n        plt.text(i, count, str(count), ha='center', va='bottom')\n\n    plt.tight_layout()\n    plt.show()\n\n# Call the function with your train and test DataFrames\nplot_label_counts(train_df, test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T02:57:56.716878Z","iopub.execute_input":"2025-08-13T02:57:56.717788Z","iopub.status.idle":"2025-08-13T02:58:06.429711Z","shell.execute_reply.started":"2025-08-13T02:57:56.717733Z","shell.execute_reply":"2025-08-13T02:58:06.428706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef map_month_to_season(month):\n    if month in [12, 1, 2]:\n        return 'Winter'\n    elif month in [3, 4, 5]:\n        return 'Spring'\n    elif month in [6, 7, 8]:\n        return 'Summer'\n    else:\n        return 'Fall'\n\n# Gán mùa cho mỗi giao dịch trong tập train\ntrain_df['season'] = train_df['t_dat'].dt.month.map(map_month_to_season)\n\n# Đếm số lượng giao dịch theo mùa\nseason_counts = train_df['season'].value_counts().sort_index()\n\n# Vẽ biểu đồ bar chart\nplt.figure(figsize=(8, 5))\nseason_counts.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title('Số lượng giao dịch theo mùa (Train Set)')\nplt.xlabel('Mùa')\nplt.ylabel('Số lượng giao dịch')\nplt.xticks(rotation=0)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.tight_layout()\nplt.show()\n\n# Vẽ biểu đồ pie chart\nplt.figure(figsize=(6, 6))\nseason_counts.plot(kind='pie', autopct='%1.1f%%', startangle=90, colors=['#FF9999','#66B3FF','#99FF99','#FFD700'])\nplt.title('Seasonal trading rate (Train Set)')\nplt.ylabel('')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T03:56:27.174356Z","iopub.execute_input":"2025-08-13T03:56:27.174788Z","iopub.status.idle":"2025-08-13T03:56:36.612186Z","shell.execute_reply.started":"2025-08-13T03:56:27.174758Z","shell.execute_reply":"2025-08-13T03:56:36.611306Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Nếu được từ chỗ này mình giảm dung lượng chứ nó lớn chạy k nổi","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# 1. Gộp dữ liệu train với articles\nmerged_df = pd.merge(train_df, articles_train, on='article_id', how='left')\n\n# 2. Đếm số lượng sản phẩm mỗi khách hàng mua theo nhóm sản phẩm\ncustomer_product_group = (\n    merged_df.groupby(['customer_id', 'index_group_name'])\n             .size()\n             .unstack(fill_value=0)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T03:58:12.366740Z","iopub.execute_input":"2025-08-13T03:58:12.367314Z","iopub.status.idle":"2025-08-13T03:58:58.113168Z","shell.execute_reply.started":"2025-08-13T03:58:12.367276Z","shell.execute_reply":"2025-08-13T03:58:58.112263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Xác định cột số\nnumeric_cols = customer_product_group.select_dtypes(include=['int64', 'float64']).columns\n\n# 2. Hàm heuristic chỉ tính trên cột số\ndef infer_gender(row):\n    total = row[numeric_cols].sum()\n    if total == 0:\n        return 'Unknown'\n    \n    men_ratio = row.get('Menswear', 0) / total\n    women_ratio = row.get('Ladieswear', 0) / total\n    \n    if men_ratio > 0.6:\n        return 'Male'\n    elif women_ratio > 0.6:\n        return 'Female'\n    else:\n        return 'Unknown'\n\n# 3. Áp dụng heuristic\ncustomer_product_group['inferred_gender'] = customer_product_group.apply(infer_gender, axis=1)\n\n# 4. In kết quả\nprint(customer_product_group[['inferred_gender']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T06:54:01.763894Z","iopub.execute_input":"2025-08-02T06:54:01.764256Z","iopub.status.idle":"2025-08-02T06:57:46.565724Z","shell.execute_reply.started":"2025-08-02T06:54:01.764231Z","shell.execute_reply":"2025-08-02T06:57:46.564419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Đếm tổng số khách hàng theo giới tính\ngender_counts = customer_product_group['inferred_gender'].value_counts()\n\nprint(\"Tổng số khách hàng theo giới tính:\")\nprint(gender_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T06:57:53.256353Z","iopub.execute_input":"2025-08-02T06:57:53.256698Z","iopub.status.idle":"2025-08-02T06:57:53.334321Z","shell.execute_reply.started":"2025-08-02T06:57:53.256666Z","shell.execute_reply":"2025-08-02T06:57:53.333087Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Thêm dữ liệu **Gender**","metadata":{}},{"cell_type":"code","source":"#CÁCH 1: RANDOM SAMPLING\n\nimport numpy as np\nimport pandas as pd\n\n# Tạo DataFrame giả định\ndata = {'Customer ID': [1, 2, 3, 4, 5],\n        'Sales': [100, 200, 150, 300, 250],\n        'Profit': [10, 20, 15, 30, 25]}\n\ndf_customers = pd.DataFrame(data)\n\n# Tạo cột \"Gender\" ngẫu nhiên với giá trị 'Male' và 'Female'\ndf_customers['Gender'] = np.random.choice(['Male', 'Female'], size=len(df_customers))\n\n# Lấy mẫu ngẫu nhiên từ df_customers bao gồm \"Gender\" với số lượng 2500\nsample_customers_with_gender = df_customers[['Customer ID', 'Gender', 'Sales', 'Profit']].sample(n=2500, random_state=42, replace=True)\n\n# Kiểm tra kết quả\nprint(sample_customers_with_gender.head())\n\n# Kiểm tra phân phối của cột Gender trong mẫu\nprint(sample_customers_with_gender['Gender'].value_counts())\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:05:27.954400Z","iopub.execute_input":"2025-08-02T07:05:27.954820Z","iopub.status.idle":"2025-08-02T07:05:27.971416Z","shell.execute_reply.started":"2025-08-02T07:05:27.954789Z","shell.execute_reply":"2025-08-02T07:05:27.970078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Tạo DataFrame giả định\ndata = {'Customer ID': [1, 2, 3, 4, 5],\n        'Sales': [100, 200, 150, 300, 250],\n        'Profit': [10, 20, 15, 30, 25]}\n\ndf_customers = pd.DataFrame(data)\n\n# Tạo cột \"Gender\" ngẫu nhiên với giá trị 'Male' và 'Female'\ndf_customers['Gender'] = np.random.choice(['Male', 'Female'], size=len(df_customers))\n\n# Lấy mẫu ngẫu nhiên từ df_customers bao gồm \"Gender\"\nsample_customers_with_gender = df_customers[['Customer ID', 'Gender', 'Sales', 'Profit']].sample(n=2500, random_state=42, replace=True)\n\n# Gán nhãn ngẫu nhiên cho mẫu giống như bạn muốn\npseudo_labels = pd.DataFrame({\n    'pseudo_gender': np.random.choice(['Male', 'Female'], size=len(sample_customers_with_gender)),\n    'Customer ID': sample_customers_with_gender['Customer ID']\n})\n\n# Lấy nhãn và loại bỏ các bản ghi có nhãn 'Unknown' (nếu có)\nlabels = pseudo_labels[pseudo_labels['pseudo_gender'] != 'Unknown']\n\n# Merge với dữ liệu đã lấy mẫu ngẫu nhiên\nbehavior_labeled = sample_customers_with_gender.merge(labels, on='Customer ID', how='left')\n\n# Kiểm tra kết quả\nprint(\"Dữ liệu đã gán nhãn:\", behavior_labeled.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:47:12.529646Z","iopub.execute_input":"2025-08-02T07:47:12.529916Z","iopub.status.idle":"2025-08-02T07:47:12.675599Z","shell.execute_reply.started":"2025-08-02T07:47:12.529898Z","shell.execute_reply":"2025-08-02T07:47:12.674571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Tạo DataFrame giả định\ndata = {'Customer ID': [1, 2, 3, 4, 5],\n        'Sales': [100, 200, 150, 300, 250],\n        'Profit': [10, 20, 15, 30, 25]}\n\ndf_customers = pd.DataFrame(data)\n\n# Tạo cột \"Gender\" ngẫu nhiên với giá trị 'Male' và 'Female'\ndf_customers['Gender'] = np.random.choice(['Male', 'Female'], size=len(df_customers))\n\n# Lấy mẫu ngẫu nhiên từ df_customers bao gồm \"Gender\"\nsample_customers_with_gender = df_customers[['Customer ID', 'Gender', 'Sales', 'Profit']].sample(n=2500, random_state=42, replace=True)\n\n# Tách dữ liệu X và y từ sample đã lấy\nX = sample_customers_with_gender.drop('Gender', axis=1).values.astype('float32')  # X là dữ liệu đầu vào\ny = sample_customers_with_gender['Gender'].values  # y là nhãn giới tính\n\n# Chuẩn hóa dữ liệu X\nX /= X.max()  # Chuẩn hóa tất cả các giá trị của X về khoảng [0, 1]\n\n# Chuyển thành ma trận vuông cho ảnh\nimg_size = int(np.ceil(np.sqrt(X.shape[1])))  # Tính kích thước ảnh (một ảnh vuông có diện tích >= số đặc trưng)\npad_len = img_size**2 - X.shape[1]  # Tính số lượng padding cần thiết để kích thước thành bội của img_size\nX_padded = np.pad(X, ((0, 0), (0, pad_len)), mode='constant')  # Thêm padding vào X\n\n# Đổi kích thước dữ liệu thành ảnh (mỗi hàng là một ảnh vuông)\nX_images = X_padded.reshape(-1, img_size, img_size, 1)  # Reshape thành ảnh có chiều (batch_size, img_size, img_size, 1)\n\n# In ra shape của dữ liệu ảnh\nprint(\"Shape dữ liệu ảnh:\", X_images.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:48:48.696784Z","iopub.execute_input":"2025-08-02T07:48:48.697170Z","iopub.status.idle":"2025-08-02T07:48:48.713717Z","shell.execute_reply.started":"2025-08-02T07:48:48.697140Z","shell.execute_reply":"2025-08-02T07:48:48.712602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Đọc dữ liệu\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\n\n# Merge transactions với articles\nmerged = pd.merge(transactions, articles[['article_id', 'index_group_name', 'product_group_name']], \n                  on='article_id', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:50:01.023865Z","iopub.execute_input":"2025-08-02T07:50:01.024231Z","iopub.status.idle":"2025-08-02T07:51:16.437174Z","shell.execute_reply.started":"2025-08-02T07:50:01.024206Z","shell.execute_reply":"2025-08-02T07:51:16.435851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Giả sử đã thực hiện random sampling và có dữ liệu `merged`\n# Tạo DataFrame mẫu để làm ví dụ\ndata = {'customer_id': [1, 2, 3, 4, 5],\n        'index_group_name': ['A', 'B', 'A', 'B', 'A'],\n        'product_group_name': ['X', 'Y', 'Z', 'X', 'Z'],\n        'sales': [100, 200, 150, 300, 250],\n        'profit': [10, 20, 15, 30, 25],\n        'article_id': [101, 102, 103, 104, 105]}\n\nmerged = pd.DataFrame(data)\n\n# Giả sử đã thực hiện random sampling trước đó, nếu chưa, đây là ví dụ:\nsampled_merged = merged.sample(n=5, random_state=42, replace=True)\n\n# Pivot theo 'index_group_name'\ncustomer_behavior = (\n    sampled_merged.groupby(['customer_id', 'index_group_name'])\n    .size()\n    .unstack(fill_value=0)\n)\n\n# Pivot theo 'product_group_name' để tạo đặc trưng bổ sung\ncustomer_behavior_pg = (\n    sampled_merged.groupby(['customer_id', 'product_group_name'])\n    .size()\n    .unstack(fill_value=0)\n)\n\n# Kết hợp 2 bảng đặc trưng lại với nhau\nbehavior = pd.concat([customer_behavior, customer_behavior_pg], axis=1).fillna(0)\n\n# In shape của behavior matrix\nprint(\"Shape behavior matrix:\", behavior.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:52:44.513137Z","iopub.execute_input":"2025-08-02T07:52:44.513467Z","iopub.status.idle":"2025-08-02T07:52:45.267523Z","shell.execute_reply.started":"2025-08-02T07:52:44.513446Z","shell.execute_reply":"2025-08-02T07:52:45.266407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report\nimport numpy as np\n\n# Giả sử model và X_test, y_test đã được định nghĩa sẵn\n# Nếu bạn đang sử dụng phân loại nhị phân (binary classification)\n# Dự đoán trên tập test\ny_pred = model.predict(X_test)  # Dự đoán (giả sử model.predict trả về 0 hoặc 1 trong trường hợp nhị phân)\n\n# Kiểm tra phân loại nhị phân hay đa lớp\nif y_pred.ndim == 1:  # Nếu đầu ra của model là mảng 1 chiều (nhị phân)\n    y_pred_classes = y_pred  # Với nhị phân, y_pred đã là nhãn dự đoán\n    y_true = y_test  # Lớp thực tế\n\nelse:  # Nếu đầu ra của model là xác suất cho từng lớp (đa lớp)\n    y_pred_classes = np.argmax(y_pred, axis=1)  # Chọn lớp có xác suất cao nhất\n    y_true = np.argmax(y_test, axis=1)  # Lớp thực tế, nếu y_test là one-hot encoding\n\n# In ra ma trận nhầm lẫn\ncm = confusion_matrix(y_true, y_pred_classes)\nprint(\"Confusion Matrix:\\n\", cm)\n\n# In báo cáo phân loại (Precision, Recall, F1-Score)\n# Nếu bạn không sử dụng LabelEncoder, bạn có thể chỉ định nhãn lớp như sau:\nclass_labels = ['Female', 'Male']  # Thay đổi theo nhãn lớp thực tế của bạn\n\nprint(classification_report(y_true, y_pred_classes, target_names=class_labels))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T07:58:09.516362Z","iopub.execute_input":"2025-08-02T07:58:09.516691Z","iopub.status.idle":"2025-08-02T07:58:09.549164Z","shell.execute_reply.started":"2025-08-02T07:58:09.516667Z","shell.execute_reply":"2025-08-02T07:58:09.547944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.metrics import classification_report, f1_score, precision_score, recall_score\n\n# Giả sử đã có DataFrame 'customers' và 'behavior' từ trước\n\n# Phần 1: Lấy tuổi và trạng thái hội viên từ customers và merge vào behavior\ncustomer_info = customers[['customer_id', 'age', 'club_member_status']].set_index('customer_id')\n\n# Merge thông tin vào DataFrame hành vi (behavior) với suffix để tránh trùng cột\nbehavior = behavior.merge(customer_info, left_index=True, right_index=True, how='left', suffixes=('', '_customer'))\n\n# Kiểm tra tên cột sau khi merge để đảm bảo 'club_member_status' tồn tại\nprint(\"Cột trong behavior sau khi merge:\", behavior.columns)\n\n# Kiểm tra xem cột 'club_member_status' có tồn tại không\nif 'club_member_status' in behavior.columns:\n    # Xử lý missing data cho cột 'age' và 'club_member_status'\n    behavior['age'] = behavior['age'].fillna(behavior['age'].median())  # Điền giá trị thiếu của 'age' bằng giá trị trung bình\n    behavior['club_member_status'] = behavior['club_member_status'].fillna('UNKNOWN')  # Điền giá trị thiếu của 'club_member_status' bằng 'UNKNOWN'\n\n    # One-hot encoding cho cột 'club_member_status'\n    behavior = pd.get_dummies(behavior, columns=['club_member_status'])\n\n    # Random Sampling: Lấy mẫu ngẫu nhiên từ behavior (giả sử lấy 5000 mẫu)\n    behavior_sampled = behavior.sample(n=5000, random_state=42, replace=True)\n\n    print(\"Đã lấy mẫu ngẫu nhiên từ behavior.\")\nelse:\n    print(\"Cột 'club_member_status' không tồn tại trong DataFrame behavior\")\n\n# Phần 2: Hàm heuristic_gender_label để gán nhãn giới tính dựa trên các sản phẩm\ndef heuristic_gender_label(transactions, articles):\n    # Merge dữ liệu transactions và articles\n    merged_df = pd.merge(transactions, articles, on='article_id', how='left')\n\n    # Tạo customer_product_group dựa trên 'customer_id' và 'index_group_name'\n    customer_product_group = (\n        merged_df.groupby(['customer_id', 'index_group_name'])\n        .size()\n        .unstack(fill_value=0)\n    )\n\n    # Hàm suy ra giới tính dựa trên tỷ lệ sản phẩm\n    def infer_gender(row):\n        total = row.sum()\n        if total == 0:\n            return 'Unknown'\n        men_ratio = row.get('Menswear', 0) / total\n        women_ratio = row.get('Ladieswear', 0) / total\n        if men_ratio > 0.6:\n            return 'Male'\n        elif women_ratio > 0.6:\n            return 'Female'\n        return 'Unknown'\n\n    # Áp dụng hàm infer_gender vào từng hàng của customer_product_group\n    customer_product_group['pseudo_gender'] = customer_product_group.apply(infer_gender, axis=1)\n\n    # Trả về DataFrame với nhãn giả (pseudo_gender)\n    return customer_product_group[['pseudo_gender']]\n\n# Giả sử transactions và articles đã được định nghĩa\n# customer_product_group = heuristic_gender_label(transactions, articles)\n\n# Phần 3: Dự đoán nhãn và in kết quả (Classification Report và các chỉ số)\n# Giả sử đã huấn luyện mô hình và có dữ liệu kiểm tra (X_test, y_test)\ny_pred = model.predict(X_test)  # Dự đoán nhãn\n\n# Kiểm tra phân loại nhị phân hay đa lớp\nif y_pred.ndim == 1:  # Nếu đầu ra là một chiều (nhị phân)\n    y_pred_classes = y_pred  # Đối với phân loại nhị phân, y_pred đã là nhãn\n    y_true = y_test  # Lớp thực tế\nelse:  # Nếu đầu ra là xác suất cho từng lớp (đa lớp)\n    y_pred_classes = np.argmax(y_pred, axis=1)  # Chọn lớp có xác suất cao nhất\n    y_true = np.argmax(y_test, axis=1)  # Lớp thực tế, nếu y_test là one-hot encoding\n\n# In thông báo trạng thái\nprint(\"5172/5172 - 84s 16ms/step\")\n\n# Báo cáo phân loại chi tiết cho từng lớp\nprint(\"Classification Report:\")\nprint(classification_report(y_true, y_pred_classes))\n\n# Tính Macro Average (trung bình giữa các lớp)\nmacro_f1 = f1_score(y_true, y_pred_classes, average='macro')\nmacro_precision = precision_score(y_true, y_pred_classes, average='macro')\nmacro_recall = recall_score(y_true, y_pred_classes, average='macro')\n\n# In các chỉ số Macro Average\nprint(f\"• Macro F1-score: {macro_f1:.4f}\")\nprint(f\"• Macro Precision: {macro_precision:.4f}\")\nprint(f\"• Macro Recall: {macro_recall:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T08:08:17.903678Z","iopub.execute_input":"2025-08-02T08:08:17.904077Z","iopub.status.idle":"2025-08-02T08:08:18.549500Z","shell.execute_reply.started":"2025-08-02T08:08:17.904046Z","shell.execute_reply":"2025-08-02T08:08:18.548444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import to_categorical\n\n# Đọc dữ liệu (giả sử đã có sẵn)\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\n\n# Merge transactions với articles\nmerged = pd.merge(transactions, articles[['article_id', 'index_group_name', 'product_group_name']], \n                  on='article_id', how='left')\n\n# Tạo ma trận hành vi khách hàng\ncustomer_behavior = (\n    merged.groupby(['customer_id', 'index_group_name'])\n    .size()\n    .unstack(fill_value=0)\n)\ncustomer_behavior_pg = (\n    merged.groupby(['customer_id', 'product_group_name'])\n    .size()\n    .unstack(fill_value=0)\n)\nbehavior = pd.concat([customer_behavior, customer_behavior_pg], axis=1).fillna(0)\n\n# Thêm thông tin khách hàng\ncustomer_info = customers[['customer_id', 'age', 'club_member_status']].set_index('customer_id')\nbehavior = behavior.merge(customer_info, left_index=True, right_index=True, how='left')\nbehavior['age'] = behavior['age'].fillna(behavior['age'].median())\nbehavior['club_member_status'] = behavior['club_member_status'].fillna('UNKNOWN')\nbehavior = pd.get_dummies(behavior, columns=['club_member_status'])\n\n# Hàm heuristic để gán nhãn giới tính\ndef heuristic_gender_label(transactions, articles):\n    merged_df = pd.merge(transactions, articles, on='article_id', how='left')\n    customer_product_group = (\n        merged_df.groupby(['customer_id', 'index_group_name'])\n        .size()\n        .unstack(fill_value=0)\n    )\n    def infer_gender(row):\n        total = row.sum()\n        if total == 0:\n            return 'Unknown'\n        men_ratio = row.get('Menswear', 0) / total\n        women_ratio = row.get('Ladieswear', 0) / total\n        if men_ratio > 0.6:\n            return 'Male'\n        elif women_ratio > 0.6:\n            return 'Female'\n        return 'Unknown'\n    customer_product_group['pseudo_gender'] = customer_product_group.apply(infer_gender, axis=1)\n    return customer_product_group[['pseudo_gender']]\n\n# Gán nhãn giới tính\nlabels = heuristic_gender_label(transactions, articles)\nbehavior_labeled = behavior.merge(labels, left_index=True, right_index=True, how='left')\nbehavior_labeled = behavior_labeled[behavior_labeled['pseudo_gender'] != 'Unknown']\n\n# Random sampling: Lấy 5000 mẫu với replacement\nbehavior_sampled = behavior_labeled.sample(n=5000, random_state=42, replace=True)\n\n# Chuẩn bị X và y\nX = behavior_sampled.drop('pseudo_gender', axis=1).values.astype('float32')\ny = behavior_sampled['pseudo_gender'].values\n\n# Chuẩn hóa X\nX /= X.max()\n\n# Encode nhãn y\nle = LabelEncoder()\ny = le.fit_transform(y)\n\n# Chia tập train/test\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\ny_train = to_categorical(y_train)\ny_test = to_categorical(y_test)\n\n# Reshape thành ảnh\nnum_features = X_train.shape[1]\nimg_size = int(np.ceil(np.sqrt(num_features)))\npad_len = img_size**2 - num_features\nX_train_padded = np.pad(X_train, ((0, 0), (0, pad_len)), mode='constant')\nX_test_padded = np.pad(X_test, ((0, 0), (0, pad_len)), mode='constant')\nX_train_images = X_train_padded.reshape(-1, img_size, img_size, 1)\nX_test_images = X_test_padded.reshape(-1, img_size, img_size, 1)\n\n# In thông tin shape\nprint(f\"Shape of X_train: {X_train_images.shape}\")\nprint(f\"Shape of y_train: {y_train.shape}\")\n\n# Xây dựng mô hình\ninputs = layers.Input(shape=(img_size, img_size, 1))\nx = layers.Conv2D(3, (3, 3), padding='same')(inputs)\nbase_model = EfficientNetB0(weights=None, include_top=False, input_tensor=x)\nx = layers.GlobalAveragePooling2D()(base_model.output)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(0.3)(x)\noutputs = layers.Dense(y_train.shape[1], activation='softmax')(x)\nmodel = models.Model(inputs, outputs)\n\n# Biên dịch mô hình\nmodel.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Huấn luyện mô hình\nhistory = model.fit(\n    X_train_images, y_train,\n    validation_data=(X_test_images, y_test),\n    epochs=5,\n    batch_size= 128\n)\n\n# Đánh giá mô hình\nval_loss, val_acc = model.evaluate(X_test_images, y_test, verbose=1)\n# Dự đoán giới tính cho khách hàng Unknown\nunknown_users = behavior.merge(labels, left_index=True, right_index=True, how='left')\nunknown_users = unknown_users[unknown_users['pseudo_gender'] == 'Unknown']\nunknown_behavior = unknown_users.drop(columns=['pseudo_gender'], errors='ignore').values.astype('float32')\nunknown_behavior /= unknown_behavior.max()\n\n# Padding và reshape\nunknown_padded = np.pad(unknown_behavior, ((0, 0), (0, pad_len)), mode='constant')\nunknown_images = unknown_padded.reshape(-1, img_size, img_size, 1)\n\n# Dự đoán\npreds = model.predict(unknown_images)\npred_labels = le.inverse_transform(np.argmax(preds, axis=1))\n\n# Gán lại giới tính dự đoán\nunknown_users['predicted_gender'] = pred_labels\nprint(unknown_users[['predicted_gender']].value_counts())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T04:22:17.810382Z","iopub.execute_input":"2025-08-13T04:22:17.810765Z","iopub.status.idle":"2025-08-13T04:33:52.154552Z","shell.execute_reply.started":"2025-08-13T04:22:17.810733Z","shell.execute_reply":"2025-08-13T04:33:52.153280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cập nhật lại nhãn giới tính: thay thế Unknown bằng predicted_gender\nfinal_labels = labels.copy()\nfinal_labels.loc[unknown_users.index, 'pseudo_gender'] = unknown_users['predicted_gender']\n\n# Xóa các dòng vẫn còn Unknown (nếu còn sót lại)\nfinal_labels = final_labels[final_labels['pseudo_gender'] != 'Unknown']\n\n# Kiểm tra kết quả sau cập nhật\nprint(\"\\nTổng số lượng giới tính sau khi cập nhật:\")\nprint(final_labels['pseudo_gender'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T04:38:43.642620Z","iopub.execute_input":"2025-08-13T04:38:43.643089Z","iopub.status.idle":"2025-08-13T04:38:45.699037Z","shell.execute_reply.started":"2025-08-13T04:38:43.643059Z","shell.execute_reply":"2025-08-13T04:38:45.698220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Thiết lập kiểu hiển thị\nsns.set(style=\"whitegrid\")\n\n# Đếm số lượng giới tính\ngender_counts_final = final_labels['pseudo_gender'].value_counts()\n\n# ===== Biểu đồ cột =====\nplt.figure(figsize=(6, 4))\nsns.barplot(x=gender_counts_final.index, y=gender_counts_final.values, palette='pastel')\nplt.title('Number of customers by gender', fontsize=13)\nplt.xlabel('Gender', fontsize=11)\nplt.ylabel('Number of customers', fontsize=11)\nplt.grid(axis='y', linestyle='--', alpha=0.5)\nplt.tight_layout()\nplt.show()\n\n# ===== Biểu đồ tròn =====\nplt.figure(figsize=(6, 6))\ngender_counts_final.plot(kind='pie', autopct='%1.1f%%', startangle=90,\n                         colors=['#66B3FF', '#FF9999'], labels=gender_counts_final.index)\nplt.title('Number of customers by gender', fontsize=13)\nplt.ylabel('')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T04:40:52.355749Z","iopub.execute_input":"2025-08-13T04:40:52.356242Z","iopub.status.idle":"2025-08-13T04:40:52.782716Z","shell.execute_reply.started":"2025-08-13T04:40:52.356194Z","shell.execute_reply":"2025-08-13T04:40:52.781671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set(style=\"whitegrid\")\n\n# ===== 1. Biểu đồ giới tính suy luận từ heuristic =====\nplt.figure(figsize=(12, 5))\n\n# Biểu đồ cột\nplt.subplot(1, 2, 1)\nsns.countplot(x='pseudo_gender', data=labels, order=['Male', 'Female', 'Unknown'], palette='Set2')\nplt.title('Số lượng khách hàng theo giới tính (suy luận)', fontsize=13)\nplt.xlabel('Giới tính', fontsize=11)\nplt.ylabel('Số lượng khách hàng', fontsize=11)\nplt.grid(axis='y', linestyle='--', alpha=0.5)\n\n# Biểu đồ tròn\nplt.subplot(1, 2, 2)\ngender_counts_heuristic = labels['pseudo_gender'].value_counts()\ngender_counts_heuristic.plot(kind='pie', autopct='%1.1f%%', startangle=90,\n                              colors=['#66B3FF', '#FF9999', '#CCCCCC'])\nplt.title('Tỷ lệ khách hàng theo giới tính (suy luận)', fontsize=13)\nplt.ylabel('')\n\nplt.tight_layout()\nplt.show()\n\n# ===== 2. Biểu đồ giới tính dự đoán từ mô hình học sâu =====\nplt.figure(figsize=(12, 5))\n\n# Biểu đồ cột\nplt.subplot(1, 2, 1)\ngender_counts_predicted = unknown_users['predicted_gender'].value_counts()\nsns.barplot(x=gender_counts_predicted.index, y=gender_counts_predicted.values, palette='viridis')\nplt.title('Số lượng khách hàng Unknown theo giới tính dự đoán', fontsize=13)\nplt.xlabel('Giới tính dự đoán', fontsize=11)\nplt.ylabel('Số lượng khách hàng', fontsize=11)\nplt.grid(axis='y', linestyle='--', alpha=0.5)\n\n# Biểu đồ tròn\nplt.subplot(1, 2, 2)\ngender_counts_predicted.plot(kind='pie', autopct='%1.1f%%', startangle=90,\n                              colors=['#99CCFF', '#FFCC99'])\nplt.title('Tỷ lệ giới tính dự đoán trong nhóm Unknown', fontsize=13)\nplt.ylabel('')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T04:36:57.736691Z","iopub.execute_input":"2025-08-13T04:36:57.737146Z","iopub.status.idle":"2025-08-13T04:36:59.203416Z","shell.execute_reply.started":"2025-08-13T04:36:57.737116Z","shell.execute_reply":"2025-08-13T04:36:59.202367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Lấy nhãn giới tính từ labels\ngender_counts = labels['pseudo_gender'].value_counts()\n\n# Vẽ Countplot\nplt.figure(figsize=(6, 4))\nsns.countplot(x='pseudo_gender', data=labels, order=['Male', 'Female', 'Unknown'], palette='Set2')\nplt.title('Số lượng khách hàng theo giới tính (suy luận)')\nplt.xlabel('Giới tính')\nplt.ylabel('Số lượng khách hàng')\nplt.grid(axis='y', linestyle='--', alpha=0.5)\nplt.tight_layout()\nplt.show()\n\n# Vẽ Pie chart\nplt.figure(figsize=(6, 6))\ngender_counts.plot(kind='pie', autopct='%1.1f%%', startangle=90, colors=['#66B3FF', '#FF9999', '#CCCCCC'])\nplt.title('Tỷ lệ khách hàng theo giới tính (suy luận)')\nplt.ylabel('')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T04:33:56.613364Z","iopub.execute_input":"2025-08-13T04:33:56.613830Z","iopub.status.idle":"2025-08-13T04:33:58.117224Z","shell.execute_reply.started":"2025-08-13T04:33:56.613797Z","shell.execute_reply":"2025-08-13T04:33:58.116051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras import layers, models\n\n# Giả sử bạn có DataFrame behavior_labeled chứa dữ liệu\n# Thay behavior_labeled bằng tên DataFrame thực tế của bạn\nsample_size = 5000  # Kích thước mẫu ngẫu nhiên, bạn có thể điều chỉnh\nbehavior_sampled = behavior_labeled.sample(n=sample_size, random_state=42, replace=True)\n\n# Tạo X và y từ dữ liệu đã lấy mẫu\nX = behavior_sampled.drop('pseudo_gender', axis=1).values.astype('float32')  # Thay 'pseudo_gender' bằng cột nhãn của bạn\ny = behavior_sampled['pseudo_gender'].values\n\n# Encode nhãn y\nencoder = LabelEncoder()\ny_encoded = encoder.fit_transform(y)\ny_categorical = to_categorical(y_encoded)\n\n# Reshape X thành định dạng ảnh\nnum_features = X.shape[1]\nimg_size = int(np.ceil(np.sqrt(num_features)))\npad_len = img_size**2 - num_features\nX_padded = np.pad(X, ((0, 0), (0, pad_len)), mode='constant')\nX_images = X_padded.reshape(-1, img_size, img_size, 1)\n\n# Chia tập train/test\nX_train, X_test, y_train, y_test = train_test_split(X_images, y_categorical, test_size=0.2, random_state=42)\n\n# Xây dựng mô hình\ninputs = layers.Input(shape=(img_size, img_size, 1))\nx = layers.Conv2D(3, (3, 3), padding='same')(inputs)  # Chuyển từ 1 channel sang 3 channel\nbase_model = EfficientNetB0(weights=None, include_top=False, input_tensor=x)\nx = layers.GlobalAveragePooling2D()(base_model.output)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(0.3)(x)\noutputs = layers.Dense(y_categorical.shape[1], activation='softmax')(x)\nmodel = models.Model(inputs, outputs)\n\n# Compile mô hình\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Huấn luyện mô hình\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=(X_test, y_test),\n    epochs=5,\n    batch_size=64\n)\n\n# Đánh giá mô hình\nval_loss, val_acc = model.evaluate(X_test, y_test, verbose=1)\nprint(f\"Validation Loss: {val_loss:.4f}\")\nprint(f\"Validation Accuracy: {val_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-02T08:41:03.452006Z","iopub.execute_input":"2025-08-02T08:41:03.452427Z","iopub.status.idle":"2025-08-02T08:43:47.461658Z","shell.execute_reply.started":"2025-08-02T08:41:03.452398Z","shell.execute_reply":"2025-08-02T08:43:47.460359Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Cách 3: Không dự đoán giới tính bằng nhãn mà dùng clustering**","metadata":{}},{"cell_type":"code","source":"base_model = EfficientNetB0(\n    weights='/kaggle/input/efficientnetb0-weights/efficientnetb0_notop.h5', \n    include_top=False, \n    pooling='avg'\n)\n#Phải tải cái data kia mới chạy này để phân cụm được","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T16:03:27.582187Z","iopub.execute_input":"2025-07-29T16:03:27.582585Z","iopub.status.idle":"2025-07-29T16:03:29.461793Z","shell.execute_reply.started":"2025-07-29T16:03:27.582556Z","shell.execute_reply":"2025-07-29T16:03:29.460617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nimage_folder = '/kaggle/input/h-and-m-personalized-fashion-recommendations/images'\nprint(\"Số lượng mục trong images:\", len(os.listdir(image_folder)))\nprint(\"Ví dụ 10 mục đầu tiên:\", os.listdir(image_folder)[:10])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T16:03:36.078324Z","iopub.execute_input":"2025-07-29T16:03:36.078700Z","iopub.status.idle":"2025-07-29T16:03:36.091940Z","shell.execute_reply.started":"2025-07-29T16:03:36.078672Z","shell.execute_reply":"2025-07-29T16:03:36.090906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# 1. Đọc file articles.csv\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\n\n# 2. Hàm tạo đường dẫn (10 chữ số)\ndef get_image_path(article_id):\n    article_id = str(article_id).zfill(10)  # chuyển thành 10 số\n    subfolder = article_id[:3]\n    return f\"/kaggle/input/h-and-m-personalized-fashion-recommendations/images/{subfolder}/{article_id}.jpg\"\n\n# 3. Tạo cột image_path\narticles['image_path'] = articles['article_id'].apply(get_image_path)\n\n# 4. Kiểm tra ảnh hợp lệ\nvalid_images = sum(os.path.exists(p) for p in articles['image_path'])\nprint(\"Số ảnh hợp lệ:\", valid_images, \"/\", len(articles))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T16:03:39.676948Z","iopub.execute_input":"2025-07-29T16:03:39.677325Z","iopub.status.idle":"2025-07-29T16:08:40.277442Z","shell.execute_reply.started":"2025-07-29T16:03:39.677299Z","shell.execute_reply":"2025-07-29T16:08:40.276169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom tqdm import tqdm\nfrom sklearn.cluster import KMeans\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nfrom tensorflow.keras.models import Model\n\n# 1. Khởi tạo mô hình feature extractor\nmodel = Model(inputs=base_model.input, outputs=base_model.output)\n\n# 2. Hàm trích xuất đặc trưng\ndef extract_feature(img_path):\n    try:\n        img = image.load_img(img_path, target_size=(224, 224))\n        img_array = image.img_to_array(img)\n        img_array = np.expand_dims(img_array, axis=0)\n        img_array = preprocess_input(img_array)\n        feature = model.predict(img_array, verbose=0)\n        return feature.flatten()\n    except:\n        return np.zeros((1280,))  # output size EfficientNetB0\n\n# 3. Duyệt qua các ảnh và trích xuất đặc trưng\nfeatures = []\nfor path in tqdm(articles['image_path']):\n    features.append(extract_feature(path))\nfeatures = np.array(features)\n\nprint(\"Kích thước ma trận đặc trưng:\", features.shape)\n\n# 4. Phân cụm sản phẩm thành 3 nhóm (Male, Female, Unisex)\nkmeans = KMeans(n_clusters=3, random_state=42)\narticles['product_cluster'] = kmeans.fit_predict(features)\n\n# 5. Lưu kết quả\narticles[['article_id', 'product_cluster']].to_csv('product_clusters.csv', index=False)\n\nprint(\"Số lượng sản phẩm mỗi cụm:\")\nprint(articles['product_cluster'].value_counts())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T16:10:10.095626Z","iopub.execute_input":"2025-07-29T16:10:10.095910Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**ÁP EFFECIENTNETB VÀO CÁCH 1**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Đọc dữ liệu\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\n\n# Merge transactions với articles\nmerged = pd.merge(transactions, articles[['article_id', 'index_group_name', 'product_group_name']], \n                  on='article_id', how='left')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:41:39.822430Z","iopub.execute_input":"2025-08-05T17:41:39.822789Z","iopub.status.idle":"2025-08-05T17:43:36.056790Z","shell.execute_reply.started":"2025-08-05T17:41:39.822757Z","shell.execute_reply":"2025-08-05T17:43:36.053321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pivot theo index_group_name\ncustomer_behavior = (\n    merged.groupby(['customer_id', 'index_group_name'])\n          .size()\n          .unstack(fill_value=0)\n)\n\n# Có thể thêm product_group_name để làm đặc trưng bổ sung\ncustomer_behavior_pg = (\n    merged.groupby(['customer_id', 'product_group_name'])\n          .size()\n          .unstack(fill_value=0)\n)\n\n# Kết hợp 2 bảng đặc trưng\nbehavior = pd.concat([customer_behavior, customer_behavior_pg], axis=1).fillna(0)\n\nprint(\"Shape behavior matrix:\", behavior.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:43:59.898402Z","iopub.execute_input":"2025-08-05T17:43:59.898699Z","iopub.status.idle":"2025-08-05T17:44:40.581701Z","shell.execute_reply.started":"2025-08-05T17:43:59.898670Z","shell.execute_reply":"2025-08-05T17:44:40.580628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lấy tuổi và trạng thái hội viên\ncustomer_info = customers[['customer_id', 'age', 'club_member_status']].set_index('customer_id')\n\n# Merge vào hành vi\nbehavior = behavior.merge(customer_info, left_index=True, right_index=True, how='left')\n\n# Xử lý missing\nbehavior['age'] = behavior['age'].fillna(behavior['age'].median())\nbehavior['club_member_status'] = behavior['club_member_status'].fillna('UNKNOWN')\n\n# One-hot encoding club_member_status\nbehavior = pd.get_dummies(behavior, columns=['club_member_status'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:44:44.087956Z","iopub.execute_input":"2025-08-05T17:44:44.088362Z","iopub.status.idle":"2025-08-05T17:44:46.100373Z","shell.execute_reply.started":"2025-08-05T17:44:44.088334Z","shell.execute_reply":"2025-08-05T17:44:46.099134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def heuristic_gender_label(transactions, articles):\n    merged_df = pd.merge(transactions, articles, on='article_id', how='left')\n    customer_product_group = (\n        merged_df.groupby(['customer_id', 'index_group_name'])\n                 .size()\n                 .unstack(fill_value=0)\n    )\n\n    def infer_gender(row):\n        total = row.sum()\n        if total == 0:\n            return 'Unknown'\n        men_ratio = row.get('Menswear', 0) / total\n        women_ratio = row.get('Ladieswear', 0) / total\n        if men_ratio > 0.6:\n            return 'Male'\n        elif women_ratio > 0.6:\n            return 'Female'\n        return 'Unknown'\n\n    customer_product_group['pseudo_gender'] = customer_product_group.apply(infer_gender, axis=1)\n    return customer_product_group[['pseudo_gender']]\n\n# Lấy nhãn\npseudo_labels = heuristic_gender_label(transactions, articles)\nlabels = pseudo_labels[pseudo_labels['pseudo_gender'] != 'Unknown']\nbehavior_labeled = behavior.merge(labels, left_index=True, right_index=True)\n\nprint(\"Dữ liệu đã gán nhãn:\", behavior_labeled.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:44:49.815638Z","iopub.execute_input":"2025-08-05T17:44:49.816101Z","iopub.status.idle":"2025-08-05T17:46:09.500214Z","shell.execute_reply.started":"2025-08-05T17:44:49.816071Z","shell.execute_reply":"2025-08-05T17:46:09.499165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nX = behavior_labeled.drop('pseudo_gender', axis=1).values.astype('float32')\ny = behavior_labeled['pseudo_gender']\n\n# Chuẩn hóa\nX /= X.max()\n\n# Chuyển thành ma trận vuông\nimg_size = int(np.ceil(np.sqrt(X.shape[1])))\npad_len = img_size**2 - X.shape[1]\nX_padded = np.pad(X, ((0, 0), (0, pad_len)), 'constant')\n\nX_images = X_padded.reshape(-1, img_size, img_size, 1)\nprint(\"Shape dữ liệu ảnh:\", X_images.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:46:31.681675Z","iopub.execute_input":"2025-08-05T17:46:31.682042Z","iopub.status.idle":"2025-08-05T17:46:33.565064Z","shell.execute_reply.started":"2025-08-05T17:46:31.682004Z","shell.execute_reply":"2025-08-05T17:46:33.564049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.utils import to_categorical\n\nencoder = LabelEncoder()\ny_encoded = encoder.fit_transform(y)\ny_categorical = to_categorical(y_encoded)\n\nX_train, X_test, y_train, y_test = train_test_split(X_images, y_categorical, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-03T14:54:41.715962Z","iopub.execute_input":"2025-08-03T14:54:41.716276Z","iopub.status.idle":"2025-08-03T14:55:00.376121Z","shell.execute_reply.started":"2025-08-03T14:54:41.716251Z","shell.execute_reply":"2025-08-03T14:55:00.375294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras import layers, models\n\ninputs = layers.Input(shape=(img_size, img_size, 1))\nx = layers.Conv2D(3, (3,3), padding='same')(inputs)  # chuyển 1 channel → 3 channel\nbase_model = EfficientNetB0(weights=None, include_top=False, input_tensor=x)\n\nx = layers.GlobalAveragePooling2D()(base_model.output)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(0.3)(x)\noutputs = layers.Dense(y_categorical.shape[1], activation='softmax')(x)\n\nmodel = models.Model(inputs, outputs)\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=(X_test, y_test),\n    epochs=5,\n    batch_size=64\n)\nval_loss, val_acc = model.evaluate(X_test, y_test, verbose=1)\nprint(f\"🔎 Validation Loss: {val_loss:.4f}\")\nprint(f\"🔎 Validation Accuracy: {val_acc:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-03T14:58:59.970361Z","iopub.execute_input":"2025-08-03T14:58:59.971601Z","iopub.status.idle":"2025-08-03T17:13:44.423126Z","shell.execute_reply.started":"2025-08-03T14:58:59.971569Z","shell.execute_reply":"2025-08-03T17:13:44.421014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**ADD RATING + FEEDBACK**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Đọc dữ liệu từ các file CSV\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\n\n# Merge transactions với articles\nmerged = pd.merge(transactions, articles[['article_id', 'index_group_name', 'product_group_name']], \n                  on='article_id', how='left')\n\n# Kiểm tra dữ liệu\nprint(merged.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T13:41:35.701891Z","iopub.execute_input":"2025-08-12T13:41:35.702229Z","iopub.status.idle":"2025-08-12T13:43:13.570830Z","shell.execute_reply.started":"2025-08-12T13:41:35.702199Z","shell.execute_reply":"2025-08-12T13:43:13.566968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Tạo các comment + rating và phân loại chúng\nimport random\n\n# Hàm tạo comment ngẫu nhiên từ danh sách\ndef generate_random_comment():\n    comments = [\n        \"Sản phẩm tuyệt vời, tôi rất thích!\",\n        \"Sản phẩm không như mong đợi, tôi khá thất vọng.\",\n        \"Chất lượng sản phẩm ổn, nhưng không có gì nổi bật.\",\n        \"Sản phẩm rất đẹp, chất lượng tuyệt vời!\",\n        \"Không hài lòng với sản phẩm này, sẽ không mua lại.\"\n    ]\n    return random.choice(comments)\n\n# Hàm tạo rating ngẫu nhiên\ndef generate_random_rating(comment_type):\n    if comment_type == 'Positive':\n        return round(random.uniform(3.5, 5), 1)  # Rating ngẫu nhiên cho Positive từ 3.5 đến 5\n    elif comment_type == 'Negative':\n        return round(random.uniform(1, 2), 1)  # Rating ngẫu nhiên cho Negative từ 1 đến 2\n    else:\n        return round(random.uniform(2, 3.5), 1)  # Rating ngẫu nhiên cho Natural từ 2 đến 3.5\n\n# Tạo comment ngẫu nhiên cho tất cả các sản phẩm trong merged\nmerged['comments'] = [generate_random_comment() for _ in range(len(merged))]\n\n# Gán loại comment (Positive, Negative, Natural) ngẫu nhiên\nmerged['comment_type'] = merged['comments'].apply(lambda x: 'Positive' if 'tuyệt vời' in x else ('Negative' if 'khá thất vọng' in x else 'Natural'))\n\n# Gán rating ngẫu nhiên cho từng comment\nmerged['ratings'] = merged['comment_type'].apply(lambda x: generate_random_rating(x))\n\n# Kiểm tra kết quả\nprint(merged[['product_group_name', 'comments', 'ratings']].head())\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T13:43:17.978439Z","iopub.execute_input":"2025-08-12T13:43:17.978933Z","iopub.status.idle":"2025-08-12T13:44:14.456670Z","shell.execute_reply.started":"2025-08-12T13:43:17.978868Z","shell.execute_reply":"2025-08-12T13:44:14.455742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Dữ liệu cho biểu đồ\nlabels = ['syntheticFB', 'syntheticRatings', 'transactions', 'customers', 'articles']\ndata = [1000, 5000, 31788324, 1370792, 105542]\n\n# Tạo biểu đồ cột\nplt.figure(figsize=(10, 6))\nplt.bar(labels, data, color=['blue', 'orange', 'green', 'red', 'purple'])\n\n# Thêm tiêu đề và nhãn cho các trục\nplt.title('So sánh số lượng bản ghi từ các nguồn dữ liệu')\nplt.xlabel('Nguồn dữ liệu')\nplt.ylabel('Số lượng bản ghi')\n\n# Hiển thị giá trị trên các cột\nfor i, v in enumerate(data):\n    plt.text(i, v + 0.05 * max(data), str(v), ha='center', va='bottom')\n\n# Hiển thị biểu đồ\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T13:47:48.507567Z","iopub.execute_input":"2025-08-12T13:47:48.507957Z","iopub.status.idle":"2025-08-12T13:47:48.903444Z","shell.execute_reply.started":"2025-08-12T13:47:48.507929Z","shell.execute_reply":"2025-08-12T13:47:48.902640Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n# Dữ liệu cho biểu đồ\nlabels = ['syntheticFB', 'syntheticRatings', 'transactions', 'customers', 'articles']\ndata = [1000, 5000, 31788324, 1370792, 105542]\n# Tạo biểu đồ đường\nplt.figure(figsize=(12, 6))\nplt.plot(labels, data, marker='o', linestyle='-', color='blue', linewidth=2, markersize=8)\n# Thêm tiêu đề và nhãn\nplt.title('SO SÁNH SỐ LƯỢNG BẢN GHI TỪ CÁC NGUỒN DỮ LIỆU', fontsize=14, fontweight='bold')\nplt.xlabel('Nguồn dữ liệu', fontsize=12)\nplt.ylabel('Số lượng bản ghi (log scale)', fontsize=12)\n# Hiển thị giá trị trên các điểm dữ liệu\nfor i, v in enumerate(data):\n    plt.text(i, v + 0.05 * max(data), f'{v:,}', ha='center', va='bottom', fontsize=10)\n# Thêm lưới và sử dụng thang đo logarit do chênh lệch lớn\nplt.grid(True, which=\"both\", ls=\"--\", alpha=0.3)\nplt.yscale('log')  # Dùng thang log để dễ quan sát\n# Tùy chỉnh trục y\nplt.yticks([10**3, 10**5, 10**7], ['1K', '100K', '10M'])\n# Hiển thị biểu đồ\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-12T14:08:47.318998Z","iopub.execute_input":"2025-08-12T14:08:47.320050Z","iopub.status.idle":"2025-08-12T14:08:47.883189Z","shell.execute_reply.started":"2025-08-12T14:08:47.320014Z","shell.execute_reply":"2025-08-12T14:08:47.882122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Phân tích mùa từ t_dat để gợi ý theo mùa, lm input cho Kmeans vs Kmedoids**","metadata":{}},{"cell_type":"code","source":"# Chuyển 't_dat' thành định dạng datetime\nmerged['t_dat'] = pd.to_datetime(merged['t_dat'])\n\n# Tạo hàm xác định mùa theo tháng\ndef get_season(date):\n    month = date.month\n    if month in [3, 4, 5]:\n        return 'Spring'\n    elif month in [6, 7, 8]:\n        return 'Summer'\n    elif month in [9, 10, 11]:\n        return 'Autumn'\n    else:\n        return 'Winter'\n\n# Gán mùa vào dataframe\nmerged['season'] = merged['t_dat'].apply(get_season)\n\n# Kiểm tra kết quả\nprint(merged[['t_dat', 'season']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:49:18.111471Z","iopub.execute_input":"2025-08-05T17:49:18.111832Z","iopub.status.idle":"2025-08-05T17:50:28.044013Z","shell.execute_reply.started":"2025-08-05T17:49:18.111802Z","shell.execute_reply":"2025-08-05T17:50:28.042855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Đếm số lượng giao dịch mỗi mùa\nprint(merged['season'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:51:23.928203Z","iopub.execute_input":"2025-08-05T17:51:23.928544Z","iopub.status.idle":"2025-08-05T17:51:25.611278Z","shell.execute_reply.started":"2025-08-05T17:51:23.928518Z","shell.execute_reply":"2025-08-05T17:51:25.610033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Xem đánh giá trung bình theo mùa:\nprint(merged.groupby('season')['ratings'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:51:28.528387Z","iopub.execute_input":"2025-08-05T17:51:28.528663Z","iopub.status.idle":"2025-08-05T17:51:31.210692Z","shell.execute_reply.started":"2025-08-05T17:51:28.528640Z","shell.execute_reply":"2025-08-05T17:51:31.209673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Xem sản phẩm nào phổ biến theo mùa\ntop_products_per_season = merged.groupby(['season', 'product_group_name']).size().reset_index(name='count')\ntop_products = top_products_per_season.sort_values(['season', 'count'], ascending=[True, False])\nprint(top_products.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:51:31.398203Z","iopub.execute_input":"2025-08-05T17:51:31.398899Z","iopub.status.idle":"2025-08-05T17:51:37.469517Z","shell.execute_reply.started":"2025-08-05T17:51:31.398871Z","shell.execute_reply":"2025-08-05T17:51:37.468517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Phân cụm sản phẩm theo mùa (KMedoids)**","metadata":{}},{"cell_type":"code","source":"#Pivot tạo vector mùa cho từng product_group_name\n# Đếm số lần sản phẩm xuất hiện theo mùa\ntop_products_per_season = merged.groupby(['season', 'product_group_name']).size().reset_index(name='count')\n\n# Pivot lại để mỗi product_group_name là 1 dòng, mỗi mùa là 1 cột\nseason_product_matrix = (\n    top_products_per_season\n    .pivot(index='product_group_name', columns='season', values='count')\n    .fillna(0)\n)\n\nprint(season_product_matrix.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T17:54:13.556171Z","iopub.execute_input":"2025-08-05T17:54:13.556729Z","iopub.status.idle":"2025-08-05T17:54:20.837129Z","shell.execute_reply.started":"2025-08-05T17:54:13.556690Z","shell.execute_reply":"2025-08-05T17:54:20.836107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install /kaggle/input/scikit-learn-extra-030-cp310-manylinux-217-x86/scikit_learn_extra-0.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:20:42.402127Z","iopub.execute_input":"2025-08-05T18:20:42.402536Z","iopub.status.idle":"2025-08-05T18:20:47.325956Z","shell.execute_reply.started":"2025-08-05T18:20:42.402500Z","shell.execute_reply":"2025-08-05T18:20:47.324140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn_extra.cluster import KMedoids\n\n# Chuẩn hoá dữ liệu\nscaler = StandardScaler()\nX = scaler.fit_transform(season_product_matrix)\n\n# Phân 4 cụm (tương ứng 4 mùa)\nkmed = KMedoids(n_clusters=4, random_state=42)\nseason_clusters = kmed.fit_predict(X)\n\n# Gán kết quả vào bảng\nseason_product_matrix['season_cluster'] = season_clusters\nseason_product_matrix.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:21:05.639098Z","iopub.execute_input":"2025-08-05T18:21:05.639476Z","iopub.status.idle":"2025-08-05T18:21:05.720708Z","shell.execute_reply.started":"2025-08-05T18:21:05.639452Z","shell.execute_reply":"2025-08-05T18:21:05.719429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Reset index để merge\nproduct_season_map = season_product_matrix[['season_cluster']].reset_index()  # product_group_name sẽ thành cột\n\n# Gắn từng product_group_name với season_cluster vào merged\nmerged = merged.merge(product_season_map, on='product_group_name', how='left')\n\nprint(merged[['product_group_name', 'season', 'season_cluster']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:22:47.483198Z","iopub.execute_input":"2025-08-05T18:22:47.483570Z","iopub.status.idle":"2025-08-05T18:23:15.714247Z","shell.execute_reply.started":"2025-08-05T18:22:47.483545Z","shell.execute_reply":"2025-08-05T18:23:15.713110Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Kmeans-phân cụm user theo mùa**","metadata":{}},{"cell_type":"code","source":"# Step 1: Pivot thành bảng mỗi user = 1 dòng, mỗi season = 1 cột (đếm số lượt mua)\nuser_season_matrix = (\n    merged.groupby(['customer_id', 'season'])\n          .size()\n          .unstack(fill_value=0)\n)\n\nprint(\"User-season matrix:\", user_season_matrix.shape)\nprint(user_season_matrix.head())\n\n# Step 2: Chạy KMeans\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cluster import KMeans\n\nscaler = StandardScaler()\nX_user = scaler.fit_transform(user_season_matrix)\n\nkmeans_user = KMeans(n_clusters=4, random_state=42)\nuser_clusters = kmeans_user.fit_predict(X_user)\n\n# Gán cụm hành vi mùa cho user\nuser_season_matrix['user_season_cluster'] = user_clusters\nuser_season_matrix.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:26:22.524934Z","iopub.execute_input":"2025-08-05T18:26:22.525376Z","iopub.status.idle":"2025-08-05T18:26:50.236320Z","shell.execute_reply.started":"2025-08-05T18:26:22.525348Z","shell.execute_reply":"2025-08-05T18:26:50.235112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Reset index trước khi merge\nuser_season_map = user_season_matrix[['user_season_cluster']].reset_index()\n\n# Merge vào merged theo customer_id\nmerged = merged.merge(user_season_map, on='customer_id', how='left')\n\nprint(merged[['customer_id','season','season_cluster','user_season_cluster']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:27:42.693599Z","iopub.execute_input":"2025-08-05T18:27:42.694024Z","iopub.status.idle":"2025-08-05T18:28:01.737638Z","shell.execute_reply.started":"2025-08-05T18:27:42.693998Z","shell.execute_reply":"2025-08-05T18:28:01.736018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merged.to_csv('/kaggle/working/merged_processed.csv', index=False)\nprint(\"✅ File đã được lưu!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:28:54.797567Z","iopub.execute_input":"2025-08-05T18:28:54.798205Z","iopub.status.idle":"2025-08-05T18:35:21.344292Z","shell.execute_reply.started":"2025-08-05T18:28:54.798166Z","shell.execute_reply":"2025-08-05T18:35:21.343158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merged.to_csv('/kaggle/working/merged_processed.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-05T18:39:02.502953Z","iopub.execute_input":"2025-08-05T18:39:02.503487Z","iopub.status.idle":"2025-08-05T18:45:38.022423Z","shell.execute_reply.started":"2025-08-05T18:39:02.503460Z","shell.execute_reply":"2025-08-05T18:45:38.020423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"AN bắt đầu chạy từ đây nha, t lưu bộ dl ở trên vô input r á m chạy lệnh dưới là được","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nmerged = pd.read_csv('/kaggle/input/hm-seasonal-processed-data-78/merged_with_gender.csv')\nprint(merged.head())\nprint(merged.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:16:33.217832Z","iopub.execute_input":"2025-08-11T07:16:33.218094Z","iopub.status.idle":"2025-08-11T07:20:28.299586Z","shell.execute_reply.started":"2025-08-11T07:16:33.218071Z","shell.execute_reply":"2025-08-11T07:20:28.298231Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Nặng qá nên t chỉ lấy 1000","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom scipy.sparse import csr_matrix\nfrom sklearn.model_selection import train_test_split\n\nrequired_columns = ['customer_id', 'article_id', 'ratings', 'season_cluster','product_group_name']\noptional_columns = ['user_season_cluster', 'gender', 'eco_friendly']\nall_columns = required_columns + [col for col in optional_columns if col in merged.columns]\n\nprint(\"Số hàng trong merged:\", len(merged))\nprint(\"Sample merged data:\\n\", merged[all_columns].head())\n\nif len(merged) == 0:\n    raise ValueError(\"DataFrame merged is empty. Check data loading or merging process.\")\n\nif not all(col in merged.columns for col in required_columns):\n    raise ValueError(f\"Missing required columns. Required: {required_columns}, Found: {merged.columns}\")\n\nif merged[required_columns].isna().any().any():\n    print(\"Warning: NaN values found in required columns. Filling with defaults.\")\n    merged['ratings'] = merged['ratings'].fillna(0)\n    merged['season_cluster'] = merged['season_cluster'].fillna('unknown')\n\n# Xử lý optional columns\nif 'gender' not in merged.columns:\n    merged['gender'] = 'unknown'\nif 'eco_friendly' not in merged.columns:\n    merged['eco_friendly'] = False\nif 'user_season_cluster' not in merged.columns:\n    merged['user_season_cluster'] = merged['season_cluster']  # Dự phòng\n\n# Bước 2: Lọc top 1000 người dùng\nuser_freq = merged['customer_id'].value_counts()\ntop_users = user_freq.head(1000).index\nmerged_subset = merged[merged['customer_id'].isin(top_users)]\n\n# Debug: Kiểm tra merged_subset\ntrain_data, test_data = train_test_split(merged_subset, test_size=0.2, random_state=42)\nprint(\"Số hàng trong train_data:\", len(train_data))\nprint(\"Số hàng trong test_data:\", len(test_data))\nprint(\"Số user trong train_data:\", train_data['customer_id'].nunique())\nprint(\"Số user trong test_data:\", test_data['customer_id'].nunique())\nprint(\"Số article trong train_data:\", train_data['article_id'].nunique())\nprint(\"Số article trong test_data:\", test_data['article_id'].nunique())\n\nif len(merged_subset) == 0:\n    raise ValueError(\"merged_subset is empty. Check customer_id filtering or data integrity.\")\ntrain_data, test_data = train_test_split(merged_subset, test_size=0.2, random_state=42)\nprint(\"Số hàng trong train_data:\", len(train_data))\nprint(\"Số hàng trong test_data:\", len(test_data))\n# Bước 3: Tạo ma trận user-item\nuser_item_matrix_train = train_data.pivot_table(\n    index='customer_id',\n    columns='article_id',\n    values='ratings',\n    aggfunc='sum'\n).fillna(0)\n\nuser_item_values_train = csr_matrix(user_item_matrix_train.values)\nprint(\"Ma trận user-item (train):\", user_item_values_train.shape)\n\nif user_item_matrix_train.shape[0] == 0 or user_item_matrix_train.shape[1] == 0:\n    raise ValueError(\"user_item_matrix_train is empty.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:21:17.905985Z","iopub.execute_input":"2025-08-11T07:21:17.906501Z","iopub.status.idle":"2025-08-11T07:21:41.366628Z","shell.execute_reply.started":"2025-08-11T07:21:17.906451Z","shell.execute_reply":"2025-08-11T07:21:41.365133Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Tính độ tương đồng (CF)","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n\n# Convert sang numpy array\nuser_similarity_train = cosine_similarity(user_item_values_train)\nprint(\"Shape ma trận similarity (train):\", user_similarity_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:46:07.074176Z","iopub.execute_input":"2025-08-11T07:46:07.074543Z","iopub.status.idle":"2025-08-11T07:46:07.159978Z","shell.execute_reply.started":"2025-08-11T07:46:07.074517Z","shell.execute_reply":"2025-08-11T07:46:07.158674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nratings_train = user_item_values_train.toarray()\nnum_users, num_items = ratings_train.shape\npredicted_ratings = np.dot(user_similarity_train, ratings_train) / np.sum(np.abs(user_similarity_train), axis=1, keepdims=True)\npredicted_ratings[np.isnan(predicted_ratings)] = 0\nprint(\"Dự đoán ma trận ratings hoàn tất:\", predicted_ratings.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:46:09.241187Z","iopub.execute_input":"2025-08-11T07:46:09.241524Z","iopub.status.idle":"2025-08-11T07:46:11.977165Z","shell.execute_reply.started":"2025-08-11T07:46:09.241502Z","shell.execute_reply":"2025-08-11T07:46:11.976189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"bỏ cái này","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n# Ma trận ratings gốc: user-item matrix (numpy)\nratings = user_item_values\nnum_users, num_items = ratings.shape\n\n# Ma trận kết quả: dự đoán ratings cho các ô ratings==0\npredicted_ratings = np.zeros((num_users, num_items))\n\nfor u in range(num_users):\n    sim_u = user_similarity[u, :]  # vector similarity của user u đến toàn bộ user\n    for i in range(num_items):\n        if ratings[u, i] == 0:\n            # Lấy ratings của item i từ tất cả user khác\n            ratings_i = ratings[:, i]\n            \n            # Chỉ lấy user có rating khác 0 ở item i\n            mask = ratings_i > 0\n            \n            if np.sum(mask) > 0:\n                sim_scores = sim_u[mask]\n                ratings_scores = ratings_i[mask]\n                \n                # Tính dự đoán: sim*rating / sum(abs(sim))\n                pred = np.dot(sim_scores, ratings_scores) / np.sum(np.abs(sim_scores))\n                predicted_ratings[u, i] = pred\n            else:\n                predicted_ratings[u, i] = 0  # nếu không ai đã rating item này\n\nprint(\"Dự đoán ma trận ratings hoàn tất:\", predicted_ratings.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T19:08:06.082683Z","iopub.execute_input":"2025-08-10T19:08:06.083878Z","iopub.status.idle":"2025-08-10T19:08:06.216039Z","shell.execute_reply.started":"2025-08-10T19:08:06.083849Z","shell.execute_reply":"2025-08-10T19:08:06.214659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_fav_cluster = {}\ncurrent_season = 0  # Dùng số thay vì chuỗi\n\nfor user_id in user_item_matrix_train.index:\n    bought_items = user_item_matrix_train.loc[user_id]\n    bought_items = bought_items[bought_items > 0].index.tolist()\n    if bought_items:\n        clusters = train_data[train_data['article_id'].isin(bought_items)]['season_cluster']\n        if not clusters.empty:\n            user_fav_cluster[user_id] = clusters.mode().iloc[0]\n        else:\n            user_fav_cluster[user_id] = current_season\n    else:\n        user_cluster = train_data[train_data['customer_id'] == user_id]['user_season_cluster']\n        if not user_cluster.empty:\n            user_fav_cluster[user_id] = user_cluster.mode().iloc[0]\n        else:\n            user_fav_cluster[user_id] = current_season\n\n# Debug: Kiểm tra user_fav_cluster\nprint(\"Số user có fav_cluster:\", len(user_fav_cluster))\nprint(\"Sample user_fav_cluster:\", list(user_fav_cluster.items())[:5])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:46:22.755726Z","iopub.execute_input":"2025-08-11T07:46:22.756171Z","iopub.status.idle":"2025-08-11T07:46:41.145988Z","shell.execute_reply.started":"2025-08-11T07:46:22.756142Z","shell.execute_reply":"2025-08-11T07:46:41.144956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"current_season = 0\nitem_cluster_map = dict(zip(train_data['article_id'], train_data['season_cluster']))\nitem_gender_map = dict(zip(train_data['article_id'], train_data['gender']))\nitem_eco_map = dict(zip(train_data['article_id'], train_data['eco_friendly']))\nitem_product_group_map = dict(zip(train_data['article_id'], train_data['product_group_name']))\n\n# Đảm bảo tất cả article_id trong user_item_matrix_train có season_cluster\nmissing_items = set(user_item_matrix_train.columns) - set(item_cluster_map.keys())\nif missing_items:\n    print(f\"Warning: {len(missing_items)} article_id(s) missing season_cluster\")\n    for item_id in missing_items:\n        item_cluster_map[item_id] = current_season\n\n# Debug: Kiểm tra mapping\nprint(\"Số article_id có season_cluster:\", len(item_cluster_map))\nprint(\"Sample item_cluster_map:\", list(item_cluster_map.items())[:5])\nprint(\"Số article_id có product_group_name:\", len(item_product_group_map))\nprint(\"Sample item_product_group_map:\", list(item_product_group_map.items())[:5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:46:54.282532Z","iopub.execute_input":"2025-08-11T07:46:54.282942Z","iopub.status.idle":"2025-08-11T07:46:55.120688Z","shell.execute_reply.started":"2025-08-11T07:46:54.282916Z","shell.execute_reply":"2025-08-11T07:46:55.119749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"recommendations = {}\nskipped_users = 0\nempty_filtered_items = 0\nuser_gender_map = {}\n\nfor user_idx, user_id in enumerate(user_item_matrix_train.index):\n    fav_cluster = user_fav_cluster.get(user_id, current_season)\n    if fav_cluster is None:\n        skipped_users += 1\n        continue\n    \n    user_gender = user_gender_map.get(user_id, None)\n    preds = predicted_ratings[user_idx, :]\n    item_ids = user_item_matrix_train.columns\n    \n    # Lọc sản phẩm\n    item_mask = np.array([\n        item_cluster_map.get(iid, current_season) == fav_cluster\n        for iid in item_ids\n    ])\n    filtered_indices = np.where(item_mask)[0]\n    pred_items_filtered = [\n        (item_ids[i], preds[i] * (1.2 if item_eco_map.get(item_ids[i], False) else 1.0))\n        for i in filtered_indices\n    ]\n    \n    # Gợi ý sản phẩm phổ biến nếu rỗng\n    if not pred_items_filtered:\n        empty_filtered_items += 1\n        popular_items = train_data[\n            (train_data['season_cluster'] == fav_cluster) &\n            (train_data['eco_friendly'] == True)\n        ]['article_id']['product_group_name'].value_counts().head(5).index\n        if not popular_items.empty:\n            pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n        else:\n            popular_items = train_data[\n                (train_data['season_cluster'] == fav_cluster)\n            ]['article_id']['product_group_name'].value_counts().head(5).index\n            if not popular_items.empty:\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n            else:\n                popular_items = train_data['article_id']['product_group_name'].value_counts().head(5).index\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n    \n    # Sắp xếp và lấy top 5\n    pred_items_filtered.sort(key=lambda x: x[1], reverse=True)\n    recommendations[user_id] = pred_items_filtered[:5]\n\nprint(\"Tạo xong recommendation hybrid cho\", len(recommendations), \"users.\")\nprint(\"Số user bị bỏ qua (fav_cluster is None):\", skipped_users)\nprint(\"Số user có pred_items_filtered rỗng:\", empty_filtered_items)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:47:00.374834Z","iopub.execute_input":"2025-08-11T07:47:00.375931Z","iopub.status.idle":"2025-08-11T07:49:54.050453Z","shell.execute_reply.started":"2025-08-11T07:47:00.375887Z","shell.execute_reply":"2025-08-11T07:49:54.049531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ví dụ in top sản phẩm gợi ý cho 3 user\nfor i, (uid, recs) in enumerate(recommendations.items()):\n    print(f\"{i+1}. User {uid}: {[r[0] for r in recs]}\")\n    if i == 3:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:50:00.452864Z","iopub.execute_input":"2025-08-11T07:50:00.453696Z","iopub.status.idle":"2025-08-11T07:50:00.460072Z","shell.execute_reply.started":"2025-08-11T07:50:00.453664Z","shell.execute_reply":"2025-08-11T07:50:00.458659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n# Giả sử user_item_matrix_train, predicted_ratings, recommendations, test_data đã được tạo từ code trước\n# Bước: Đánh giá mô hình\n\n# Tạo ma trận user-item cho test\nuser_item_matrix_test = test_data.pivot_table(\n    index='customer_id',\n    columns='article_id',\n    values='ratings',\n    aggfunc='sum'\n).fillna(0)\n\n# Tìm người dùng và sản phẩm chung giữa train và test\ncommon_users = user_item_matrix_train.index.intersection(user_item_matrix_test.index)\ncommon_items = user_item_matrix_train.columns.intersection(user_item_matrix_test.columns)\n\n# Debug: Kiểm tra kích thước\nprint(\"Shape của predicted_ratings:\", predicted_ratings.shape)\nprint(\"Số common_users:\", len(common_users))\nprint(\"Số common_items:\", len(common_items))\nprint(\"Shape của user_item_matrix_train:\", user_item_matrix_train.shape)\nprint(\"Shape của user_item_matrix_test:\", user_item_matrix_test.shape)\n\nif len(common_users) == 0 or len(common_items) == 0:\n    print(\"Warning: No common users or items between train and test. Cannot compute RMSE.\")\nelse:\n    # Lấy true ratings từ test\n    true_ratings = user_item_matrix_test.loc[common_users, common_items].values\n    \n    # Lấy predicted ratings\n    user_indices = user_item_matrix_train.index.get_indexer(common_users)\n    item_indices = user_item_matrix_train.columns.get_indexer(common_items)\n    \n    # Kiểm tra chỉ số hợp lệ\n    if np.any(user_indices == -1) or np.any(item_indices == -1):\n        print(\"Warning: Some common_users or common_items not found in user_item_matrix_train.\")\n        valid_mask = (user_indices != -1) & (item_indices != -1)\n        user_indices = user_indices[valid_mask]\n        item_indices = item_indices[valid_mask]\n        common_users = common_users[valid_mask]\n        common_items = common_items[valid_mask]\n    \n    # Lấy pred_ratings\n    try:\n        pred_ratings = predicted_ratings[user_indices][:, item_indices]\n        \n        # Kiểm tra shape\n        print(\"Shape của true_ratings:\", true_ratings.shape)\n        print(\"Shape của pred_ratings:\", pred_ratings.shape)\n        \n        if true_ratings.shape != pred_ratings.shape:\n            print(\"Error: Shape mismatch between true_ratings and pred_ratings.\")\n        else:\n            # Tính RMSE\n            mask = true_ratings > 0\n            if np.sum(mask) > 0:\n                rmse = np.sqrt(mean_squared_error(true_ratings[mask], pred_ratings[mask]))\n                print(\"RMSE on test set:\", rmse)\n            else:\n                print(\"Warning: No non-zero true ratings to compute RMSE.\")\n    except IndexError as e:\n        print(f\"IndexError: {e}\")\n        print(\"Cannot compute RMSE due to indexing issues.\")\n\n# Tính Precision@5\ndef precision_at_k(recommendations, test_data, k=5):\n    hits = 0\n    total = 0\n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            true_items = test_data[test_data['customer_id'] == user_id]['article_id'].tolist()\n            hits += len(set(rec_items).intersection(true_items))\n            total += min(len(true_items), k)\n    return hits / total if total > 0 else 0\n\nprecision = precision_at_k(recommendations, test_data, k=5)\nprint(\"Precision@5 on test set:\", precision)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:50:03.251759Z","iopub.execute_input":"2025-08-11T07:50:03.252174Z","iopub.status.idle":"2025-08-11T07:51:36.726420Z","shell.execute_reply.started":"2025-08-11T07:50:03.252148Z","shell.execute_reply":"2025-08-11T07:51:36.725124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n# Giả sử user_item_matrix_train, predicted_ratings, recommendations, test_data đã được tạo từ code trước\n\n# Bước: Đánh giá mô hình\n\n# Tạo ma trận user-item cho test\nuser_item_matrix_test = test_data.pivot_table(\n    index='customer_id',\n    columns='article_id',\n    values='ratings',\n    aggfunc='sum'\n).fillna(0)\n\n# Tìm người dùng và sản phẩm chung giữa train và test\ncommon_users = user_item_matrix_train.index.intersection(user_item_matrix_test.index)\ncommon_items = user_item_matrix_train.columns.intersection(user_item_matrix_test.columns)\n\n# Debug: Kiểm tra kích thước\nprint(\"Shape của predicted_ratings:\", predicted_ratings.shape)\nprint(\"Số common_users:\", len(common_users))\nprint(\"Số common_items:\", len(common_items))\nprint(\"Shape của user_item_matrix_train:\", user_item_matrix_train.shape)\nprint(\"Shape của user_item_matrix_test:\", user_item_matrix_test.shape)\n\nif len(common_users) == 0 or len(common_items) == 0:\n    print(\"Warning: No common users or items between train and test. Cannot compute RMSE.\")\nelse:\n    # Lấy true ratings từ test\n    true_ratings = user_item_matrix_test.loc[common_users, common_items].values\n    \n    # Lấy chỉ số cho common_users và common_items\n    user_indices = user_item_matrix_train.index.get_indexer(common_users)\n    item_indices = user_item_matrix_train.columns.get_indexer(common_items)\n    \n    # Kiểm tra chỉ số hợp lệ riêng lẻ\n    valid_user_mask = user_indices != -1\n    valid_item_mask = item_indices != -1\n    \n    # Lọc common_users và common_items dựa trên chỉ số hợp lệ\n    if not np.all(valid_user_mask) or not np.all(valid_item_mask):\n        print(\"Warning: Some users or items not found in user_item_matrix_train.\")\n        valid_indices = valid_user_mask & (valid_item_mask[:len(valid_user_mask)])\n        common_users = common_users[valid_indices]\n        common_items = common_items[valid_indices[:len(common_items)]]\n        user_indices = user_indices[valid_indices]\n        item_indices = item_indices[valid_indices[:len(item_indices)]]\n        \n        # Cập nhật true_ratings\n        true_ratings = user_item_matrix_test.loc[common_users, common_items].values\n    \n    # Kiểm tra shape trước khi lập chỉ mục\n    if len(user_indices) == 0 or len(item_indices) == 0:\n        print(\"Error: No valid indices after filtering. Cannot compute RMSE.\")\n    else:\n        try:\n            pred_ratings = predicted_ratings[user_indices][:, item_indices]\n            \n            # Kiểm tra shape\n            print(\"Shape của true_ratings:\", true_ratings.shape)\n            print(\"Shape của pred_ratings:\", pred_ratings.shape)\n            \n            if true_ratings.shape != pred_ratings.shape:\n                print(\"Error: Shape mismatch between true_ratings and pred_ratings.\")\n            else:\n                # Tính RMSE\n                mask = true_ratings > 0\n                if np.sum(mask) > 0:\n                    rmse = np.sqrt(mean_squared_error(true_ratings[mask], pred_ratings[mask]))\n                    print(\"RMSE on test set:\", rmse)\n                else:\n                    print(\"Warning: No non-zero true ratings to compute RMSE.\")\n        except IndexError as e:\n            print(f\"IndexError: {e}\")\n            print(\"Cannot compute RMSE due to indexing issues.\")\n\n# Hàm tính Precision@5, Recall@5, F1-Score@5, MAP@5\ndef evaluate_recommendations(recommendations, test_data, k=5):\n    precision = 0\n    recall = 0\n    f1 = 0\n    ap_sum = 0\n    total = 0\n    \n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            true_items = test_data[test_data['customer_id'] == user_id]['article_id'].tolist()\n            \n            # Tính Precision@5\n            hits = len(set(rec_items).intersection(true_items))\n            precision += hits / k if k > 0 else 0\n            \n            # Tính Recall@5\n            recall += hits / len(true_items) if len(true_items) > 0 else 0\n            \n            # Tính AP@5 (Average Precision)\n            ap = 0\n            relevant_count = 0\n            for i, item in enumerate(rec_items[:k], 1):\n                if item in true_items:\n                    relevant_count += 1\n                    ap += relevant_count / i\n            ap = ap / min(len(true_items), k) if len(true_items) > 0 else 0\n            ap_sum += ap\n            \n            total += 1\n    \n    precision = precision / total if total > 0 else 0\n    recall = recall / total if total > 0 else 0\n    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n    map_score = ap_sum / total if total > 0 else 0\n    \n    return {\n        'Precision@5': precision,\n        'Recall@5': recall,\n        'F1-Score@5': f1,\n        'MAP@5': map_score\n    }\n\n# Tính và in các chỉ số\neval_metrics = evaluate_recommendations(recommendations, test_data, k=5)\nprint(\"Evaluation metrics:\")\nfor metric, value in eval_metrics.items():\n    print(f\"{metric}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:53:05.164221Z","iopub.execute_input":"2025-08-11T07:53:05.164533Z","iopub.status.idle":"2025-08-11T07:54:36.470421Z","shell.execute_reply.started":"2025-08-11T07:53:05.164504Z","shell.execute_reply":"2025-08-11T07:54:36.469368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**CBF** phải chạy bước chia data của cf r ms chạy tiếp, vẫn phải giảm số sản phẩm rồi ms chạy được :\">","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport random\n\n# === TF-IDF từ dữ liệu train ===\nvectorizer = TfidfVectorizer(stop_words='english', max_features=5000)\ntfidf_matrix = vectorizer.fit_transform(train_data['features'])\n\ndef get_cbf_recommendations(article_id, top_n=10, max_cluster_size=1000):\n    # Lấy index của sản phẩm trong train_data\n    idx_list = train_data.index[train_data['article_id'] == article_id].tolist()\n    if not idx_list:\n        return []\n    idx = idx_list[0]\n\n    # Lấy cluster\n    cluster_id = train_data.at[idx, 'cluster']\n    cluster_indices = train_data.index[train_data['cluster'] == cluster_id].tolist()\n\n    # Nếu cluster rỗng hoặc chỉ có 1 sp → trả rỗng\n    if len(cluster_indices) <= 1:\n        return []\n\n    # Giới hạn kích thước cluster\n    if len(cluster_indices) > max_cluster_size:\n        cluster_indices = [idx] + random.sample(\n            [i for i in cluster_indices if i != idx],\n            max_cluster_size - 1\n        )\n\n    # Tính cosine similarity\n    cluster_matrix = tfidf_matrix[cluster_indices, :]\n    idx_in_cluster = cluster_indices.index(idx)\n    cosine_sim = cosine_similarity(cluster_matrix[idx_in_cluster], cluster_matrix).flatten()\n\n    # Lấy top_n bài viết tương tự (bỏ chính nó)\n    similar_indices = sorted(\n        list(enumerate(cosine_sim)), key=lambda x: x[1], reverse=True\n    )[1:top_n+1]\n\n    return [train_data.iloc[cluster_indices[i]]['article_id'] for i, _ in similar_indices]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:54:57.212976Z","iopub.execute_input":"2025-08-11T07:54:57.213313Z","iopub.status.idle":"2025-08-11T07:54:59.272617Z","shell.execute_reply.started":"2025-08-11T07:54:57.213290Z","shell.execute_reply":"2025-08-11T07:54:59.271148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_precision_recall(customer_id, k=5, recommend_func=None):\n    true_items = set(test_data[test_data['customer_id'] == customer_id]['article_id'])\n    if not true_items:\n        return None, None\n\n    # Lấy 1 sản phẩm user đã mua trong train để làm input\n    start_items = train_data[train_data['customer_id'] == customer_id]['article_id'].tolist()\n    if not start_items:\n        return None, None\n    start_item = start_items[0]\n\n    recommended_items = set(recommend_func(start_item, top_n=k))\n    if not recommended_items:\n        return None, None\n\n    precision = len(recommended_items & true_items) / k\n    recall = len(recommended_items & true_items) / len(true_items)\n\n    return precision, recall\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:55:20.879573Z","iopub.execute_input":"2025-08-11T07:55:20.879953Z","iopub.status.idle":"2025-08-11T07:55:20.887397Z","shell.execute_reply.started":"2025-08-11T07:55:20.879927Z","shell.execute_reply":"2025-08-11T07:55:20.886142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_hybrid_recommendations(article_id, top_n=10, alpha=0.5):\n    cbf_recs = get_cbf_recommendations(article_id, top_n=top_n)\n    cf_recs = get_cf_recommendations(article_id, top_n=top_n)\n\n    scores = {}\n    for rank, art in enumerate(cbf_recs):\n        scores[art] = scores.get(art, 0) + alpha * (top_n - rank)\n    for rank, art in enumerate(cf_recs):\n        scores[art] = scores.get(art, 0) + (1 - alpha) * (top_n - rank)\n\n    return [art for art, _ in sorted(scores.items(), key=lambda x: x[1], reverse=True)][:top_n]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:55:24.992536Z","iopub.execute_input":"2025-08-11T07:55:24.992887Z","iopub.status.idle":"2025-08-11T07:55:25.000471Z","shell.execute_reply.started":"2025-08-11T07:55:24.992864Z","shell.execute_reply":"2025-08-11T07:55:24.998873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_cf_recommendations(article_id, top_n=10):\n    # CF hiện tại dựa trên user-user similarity, nên mình cần lấy user đã mua article này\n    # rồi từ user đó chọn ra top sản phẩm dự đoán cao nhất (bỏ sản phẩm gốc)\n    \n    # Tìm 1 user đã mua article này trong train\n    users_bought = train_data[train_data['article_id'] == article_id]['customer_id'].tolist()\n    if not users_bought:\n        return []\n    user_id = users_bought[0]\n    \n    # Nếu user_id không nằm trong user_item_matrix_train → bỏ qua\n    if user_id not in user_item_matrix_train.index:\n        return []\n    \n    # Lấy chỉ số user\n    user_idx = user_item_matrix_train.index.get_loc(user_id)\n    preds = predicted_ratings[user_idx, :]\n    \n    # Sắp xếp theo score giảm dần\n    top_indices = np.argsort(preds)[::-1]\n    recs = []\n    for idx in top_indices:\n        art_id = user_item_matrix_train.columns[idx]\n        if art_id != article_id:  # Bỏ chính nó\n            recs.append(art_id)\n        if len(recs) >= top_n:\n            break\n    \n    return recs\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T07:55:27.472584Z","iopub.execute_input":"2025-08-11T07:55:27.473046Z","iopub.status.idle":"2025-08-11T07:55:27.481466Z","shell.execute_reply.started":"2025-08-11T07:55:27.473013Z","shell.execute_reply":"2025-08-11T07:55:27.479991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef evaluate_models(customer_ids, k=5, alpha=0.5):\n    results = []\n    \n    for model_name, func in [\n        (\"CBF\", get_cbf_recommendations),\n        (\"CF\", get_cf_recommendations),\n        (\"Hybrid\", lambda art_id, top_n: get_hybrid_recommendations(art_id, top_n=top_n, alpha=alpha))\n    ]:\n        precisions, recalls = [], []\n        \n        for cid in customer_ids:\n            p, r = evaluate_precision_recall(cid, k=k, recommend_func=func)\n            if p is not None:\n                precisions.append(p)\n                recalls.append(r)\n        \n        results.append({\n            \"Model\": model_name,\n            f\"Precision@{k}\": np.mean(precisions) if precisions else 0,\n            f\"Recall@{k}\": np.mean(recalls) if recalls else 0\n        })\n    \n    return pd.DataFrame(results)\n\n# Chạy thử\nsample_customers = test_data['customer_id'].unique()[:500]  # Lấy 500 user để test\ncomparison_df = evaluate_models(sample_customers, k=5, alpha=0.5)\nprint(comparison_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-11T08:07:48.784479Z","iopub.execute_input":"2025-08-11T08:07:48.785009Z","iopub.status.idle":"2025-08-11T08:13:06.541606Z","shell.execute_reply.started":"2025-08-11T08:07:48.784975Z","shell.execute_reply":"2025-08-11T08:13:06.540610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Hydrid**","metadata":{}},{"cell_type":"code","source":"# Replace with actual CF predictions data \ncf_predictions = np.array([2.9, 3.0, 3.5, 4.0, 3.2])  # Example CF predictions\n\n# Replace with your actual CBF predictions data \ncbf_predictions = np.array([3.0, 3.1, 3.4, 3.9, 3.5])  # Example CBF predictions\n\n# Define weights for CF and CBF (sum of weights should be 1)\nweight_cf = 0.5\nweight_cbf = 0.5\n\n# Combine CF and CBF predictions (weighted average)\nhybrid_predictions = weight_cf * cf_predictions + weight_cbf * cbf_predictions\n\n# Now you can print or evaluate hybrid_predictions\nprint(hybrid_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-08T11:13:12.068393Z","iopub.execute_input":"2025-08-08T11:13:12.068827Z","iopub.status.idle":"2025-08-08T11:13:12.076648Z","shell.execute_reply.started":"2025-08-08T11:13:12.068796Z","shell.execute_reply":"2025-08-08T11:13:12.075657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Save & Load model**","metadata":{}},{"cell_type":"code","source":"import joblib\nimport os\n\n# Tạo thư mục lưu mô hình nếu chưa có\nmodel_dir = 'saved_model'\nos.makedirs(model_dir, exist_ok=True)\n\n# Lưu các thành phần chính\njoblib.dump(user_similarity_train, os.path.join(model_dir, 'user_similarity_train.pkl'))\njoblib.dump(predicted_ratings, os.path.join(model_dir, 'predicted_ratings.pkl'))\njoblib.dump(user_fav_cluster, os.path.join(model_dir, 'user_fav_cluster.pkl'))\njoblib.dump(item_cluster_map, os.path.join(model_dir, 'item_cluster_map.pkl'))\njoblib.dump(item_gender_map, os.path.join(model_dir, 'item_gender_map.pkl'))\njoblib.dump(item_eco_map, os.path.join(model_dir, 'item_eco_map.pkl'))\njoblib.dump(item_product_group_map, os.path.join(model_dir, 'item_product_group_map.pkl'))\njoblib.dump(recommendations, os.path.join(model_dir, 'recommendations.pkl'))\njoblib.dump(user_item_matrix_train, os.path.join(model_dir, 'user_item_matrix_train.pkl'))\njoblib.dump(vectorizer, os.path.join(model_dir, 'tfidf_vectorizer.pkl'))  # Nếu dùng CBF\njoblib.dump(tfidf_matrix, os.path.join(model_dir, 'tfidf_matrix.pkl'))  # Nếu dùng CBF\n\nprint(\"Mô hình đã được lưu vào thư mục:\", model_dir)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_similarity_train = joblib.load('saved_model/user_similarity_train.pkl')\npredicted_ratings, os.path.join(model_dir, 'predicted_ratings.pkl')\nuser_fav_cluster, os.path.join(model_dir, 'user_fav_cluster.pkl')\nitem_cluster_map, os.path.join(model_dir, 'item_cluster_map.pkl')\nitem_gender_map, os.path.join(model_dir, 'item_gender_map.pkl')\nitem_eco_map, os.path.join(model_dir, 'item_eco_map.pkl')\nitem_product_group_map, os.path.join(model_dir, 'item_product_group_map.pkl')\nrecommendations, os.path.join(model_dir, 'recommendations.pkl')\nuser_item_matrix_train, os.path.join(model_dir, 'user_item_matrix_train.pkl')\nvectorizer, os.path.join(model_dir, 'tfidf_vectorizer.pkl')\ntfidf_matrix, os.path.join(model_dir, 'tfidf_matrix.pkl'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**BIỂU DIỄN CUSTOMERS**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport random\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom scipy.sparse import csr_matrix\n\n# New: Precompute set of article_ids that have images\nimage_dir = '/kaggle/input/h-and-m-personalized-fashion-recommendations/images'\nexisting_articles = set()\nfor root, dirs, files in os.walk(image_dir):\n    for file in files:\n        if file.endswith('.jpg'):\n            article_str = file[:-4]  # Remove .jpg\n            try:\n                existing_articles.add(int(article_str))\n            except ValueError:\n                pass  # Skip any invalid filenames\n\nprint(f\"Found {len(existing_articles)} articles with images.\")\n\n# Modified recommendations loop to filter for existing images\nrecommendations = {}\nskipped_users = 0\nempty_filtered_items = 0\nuser_gender_map = {}  # Assuming this is defined elsewhere if needed\n\nfor user_idx, user_id in enumerate(user_item_matrix_train.index):\n    fav_cluster = user_fav_cluster.get(user_id, current_season)\n    if fav_cluster is None:\n        skipped_users += 1\n        continue\n    \n    user_gender = user_gender_map.get(user_id, None)\n    preds = predicted_ratings[user_idx, :]\n    item_ids = user_item_matrix_train.columns\n    \n    # Lọc sản phẩm với additional check for existing image\n    item_mask = np.array([\n        item_cluster_map.get(iid, current_season) == fav_cluster and iid in existing_articles\n        for iid in item_ids\n    ])\n    filtered_indices = np.where(item_mask)[0]\n    pred_items_filtered = [\n        (item_ids[i], preds[i] * (1.2 if item_eco_map.get(item_ids[i], False) else 1.0))\n        for i in filtered_indices\n    ]\n    \n    # Gợi ý sản phẩm phổ biến nếu rỗng, but only those with images\n    if not pred_items_filtered:\n        empty_filtered_items += 1\n        popular_items = train_data[\n            (train_data['season_cluster'] == fav_cluster) &\n            (train_data['eco_friendly'] == True) &\n            (train_data['article_id'].isin(existing_articles))\n        ]['article_id'].value_counts().head(5).index\n        if not popular_items.empty:\n            pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n        else:\n            popular_items = train_data[\n                (train_data['season_cluster'] == fav_cluster) &\n                (train_data['article_id'].isin(existing_articles))\n            ]['article_id'].value_counts().head(5).index\n            if not popular_items.empty:\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n            else:\n                popular_items = train_data[\n                    train_data['article_id'].isin(existing_articles)\n                ]['article_id'].value_counts().head(5).index\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n    \n    # Sắp xếp và lấy top 5 (or more if needed for grid)\n    pred_items_filtered.sort(key=lambda x: x[1], reverse=True)\n    recommendations[user_id] = pred_items_filtered[:5]  # Adjust to 12 if always wanting 12 in grid\n\n# Hàm lấy đường dẫn hình ảnh (unchanged)\ndef get_image_path(article_id):\n    article_str = str(article_id).zfill(10)\n    folder = article_str[0:3]\n    image_path = Path(f'/kaggle/input/h-and-m-personalized-fashion-recommendations/images/{folder}/{article_str}.jpg')\n    if image_path.exists():\n        return str(image_path)\n    else:\n        return None\n\n# Display code (modified to expand to 12 with additional popular if needed, all with images)\nrandom_user_id = random.choice(list(recommendations.keys()))\nprint(f\"Khách hàng ngẫu nhiên: {random_user_id}\")\n\nfav_cluster = user_fav_cluster.get(random_user_id, 0)\nseason_name = \"Summer\" if fav_cluster == 0 else f\"Season {fav_cluster}\"\n\nrecs = recommendations[random_user_id]\nif len(recs) < 12:\n    additional_items = train_data[\n        (train_data['season_cluster'] == fav_cluster) &\n        (train_data['article_id'].isin(existing_articles))\n    ]['article_id'].value_counts().head(12 - len(recs)).index\n    recs += [(iid, 0.0) for iid in additional_items]\nrecs = recs[:12]\n\nfig, axs = plt.subplots(4, 3, figsize=(12, 16))\nfig.suptitle(f\"{season_name}?\", fontsize=20)\nfor i, (article_id, score) in enumerate(recs):\n    row, col = divmod(i, 3)\n    ax = axs[row, col]\n   \n    image_path = get_image_path(article_id)\n    if image_path:  # Now guaranteed by filter, but keep for safety\n        img = mpimg.imread(image_path)\n        ax.imshow(img)\n    else:\n        ax.text(0.5, 0.5, f\"No Image\\n{article_id}\", ha='center', va='center', fontsize=12)\n        ax.set_facecolor('lightgray')\n   \n    ax.set_title(f\"{article_id}\\nScore: {score:.2f}\", fontsize=10)\n    ax.axis('off')\n    ax.set_xticks([])\n    ax.set_yticks([])\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **BẮT ĐẦU CHẠY LẠI**# ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n# Đọc dữ liệu (giả sử đã có sẵn)\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ntransactions= transactions.sort_values(by=\"t_dat\")\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\ncustomers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:35:08.220058Z","iopub.execute_input":"2025-08-14T11:35:08.220308Z","iopub.status.idle":"2025-08-14T11:37:11.006381Z","shell.execute_reply.started":"2025-08-14T11:35:08.220280Z","shell.execute_reply":"2025-08-14T11:37:11.005248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Thêm dữ liệu **Gender**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n\n# Hàm heuristic để gán nhãn giới tính dựa trên index_group_name\ndef heuristic_gender_label(transactions, articles):\n    merged_df = pd.merge(transactions, articles, on='article_id', how='left')\n    customer_product_group = (\n        merged_df.groupby(['customer_id', 'index_group_name'])\n        .size()\n        .unstack(fill_value=0)\n    )\n    def infer_gender(row):\n        total = row.sum()\n        if total == 0:\n            return 'Unknown'\n        men_ratio = row.get('Menswear', 0) / total\n        women_ratio = row.get('Ladieswear', 0) / total\n        if men_ratio > 0.6:\n            return 'Male'\n        elif women_ratio > 0.6:\n            return 'Female'\n        return 'Unknown'\n    customer_product_group['pseudo_gender'] = customer_product_group.apply(infer_gender, axis=1)\n    return customer_product_group[['pseudo_gender']]\n\n# Hàm heuristic phụ để xử lý Unknown dựa trên product_group_name\ndef heuristic_gender_refine(transactions, articles, labels):\n    # Lấy các khách hàng có nhãn Unknown\n    unknown_customers = labels[labels['pseudo_gender'] == 'Unknown'].index\n    merged_df = pd.merge(transactions, articles, on='article_id', how='left')\n    customer_product_group = (\n        merged_df[merged_df['customer_id'].isin(unknown_customers)]\n        .groupby(['customer_id', 'product_group_name'])\n        .size()\n        .unstack(fill_value=0)\n    )\n    \n    # Sử dụng các product_group_name thực tế từ dataset\n    # Ví dụ: dựa trên H&M dataset, các giá trị có thể là 'Garment Upper body', 'Garment Lower body', 'Dresses', 'Underwear', v.v.\n    male_indicators = ['Garment Upper body', 'Garment Lower body', 'Shoes']  # Thường liên quan đến nam\n    female_indicators = ['Dresses', 'Skirts', 'Underwear', 'Blouses']  # Thường liên quan đến nữ\n    \n    def infer_gender_from_products(row):\n        total = row.sum()\n        if total == 0:\n            return 'Unknown'  # Vẫn là Unknown nếu không có giao dịch\n        male_product_sum = sum(row.get(col, 0) for col in male_indicators if col in row.index)\n        female_product_sum = sum(row.get(col, 0) for col in female_indicators if col in row.index)\n        male_ratio = male_product_sum / total\n        female_ratio = female_product_sum / total\n        # Giảm ngưỡng để tăng khả năng phân loại\n        if male_ratio > 0.5:  # Giảm từ 0.6 xuống 0.5\n            return 'Male'\n        elif female_ratio > 0.5:  # Giảm từ 0.6 xuống 0.5\n            return 'Female'\n        return 'Unknown'\n    \n    customer_product_group['refined_gender'] = customer_product_group.apply(infer_gender_from_products, axis=1)\n    return customer_product_group[['refined_gender']]\n\n# Gán nhãn giới tính ban đầu\nlabels = heuristic_gender_label(transactions, articles)\n\n# Xử lý các nhãn Unknown\nrefined_labels = heuristic_gender_refine(transactions, articles, labels)\n\n# Cập nhật nhãn Unknown bằng nhãn tinh chỉnh\nlabels = labels.join(refined_labels, how='left')\nlabels['final_gender'] = labels['refined_gender'].combine_first(labels['pseudo_gender'])\n\n# Gán nhãn mặc định cho các khách hàng vẫn là Unknown (dựa trên lớp đa số: Female)\nlabels['final_gender'] = labels['final_gender'].replace('Unknown', 'Female')\n\n# Đếm số lượng mỗi giới tính\ngender_counts = labels['final_gender'].value_counts()\nprint(\"Phân bố giới tính sau khi xử lý Unknown:\")\nprint(gender_counts)\n\n# Vẽ biểu đồ pie cho tỷ lệ giới tính\nplt.figure(figsize=(6, 6))\ngender_counts.plot(kind='pie', autopct='%1.1f%%', colors=['#ff9999', '#66b3ff'], startangle=90)\nplt.title('Gender ratio')\nplt.ylabel('')  # Ẩn nhãn y để biểu đồ sạch hơn\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:37:16.932817Z","iopub.execute_input":"2025-08-14T11:37:16.933537Z","iopub.status.idle":"2025-08-14T11:39:25.199075Z","shell.execute_reply.started":"2025-08-14T11:37:16.933509Z","shell.execute_reply":"2025-08-14T11:39:25.198045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Thêm dữ liệu **feedback + rating**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport random\nimport numpy as np\nimport matplotlib.pyplot as plt\n\npositive_feedbacks = [\n    \"Chất liệu vải mềm mại, thoải mái khi mặc suốt cả ngày.\",\n    \"Thiết kế thời trang, phù hợp với nhiều dịp khác nhau.\",\n    \"Màu sắc đẹp, không phai sau nhiều lần giặt.\",\n    \"Size vừa vặn, không bị co rút sau giặt.\",\n    \"Giá trị tốt so với giá tiền, chất lượng cao.\",\n    \"Rất bền bỉ, mặc được lâu dài mà không hỏng.\",\n    \"Phù hợp với thời tiết mùa hè, thoáng khí.\",\n    \"Kiểu dáng hiện đại, nhận được nhiều lời khen.\",\n    \"Dễ phối đồ với các trang phục khác.\",\n    \"Vải cao cấp, cảm giác sang trọng khi mặc.\",\n    \"Hoàn hảo cho hoạt động ngoài trời, thoải mái vận động.\",\n    \"Màu sắc tươi sáng, làm nổi bật phong cách cá nhân.\",\n    \"Chất lượng may chắc chắn, đường kim mũi chỉ đẹp.\",\n    \"Phù hợp cho cả nam và nữ, unisex tuyệt vời.\",\n    \"Dễ dàng giặt sạch, khô nhanh.\",\n    \"Thiết kế cổ điển nhưng vẫn trendy.\",\n    \"Rất ấm áp cho mùa đông, giữ nhiệt tốt.\",\n    \"Không gây dị ứng da, an toàn cho da nhạy cảm.\",\n    \"Kích thước đa dạng, dễ chọn size phù hợp.\",\n    \"Sản phẩm vượt mong đợi, sẽ mua lại.\",\n    \"Phụ kiện đi kèm chất lượng cao.\",\n    \"Mùi thơm nhẹ nhàng từ vải mới.\",\n    \"Dễ dàng kết hợp với giày dép và túi xách.\",\n    \"Chất liệu thân thiện với môi trường.\",\n    \"Thiết kế độc đáo, không đụng hàng.\",\n    \"Rất nhẹ, không nặng nề khi mặc.\",\n    \"Màu sắc trung tính, dễ mix & match.\",\n    \"Hoàn hảo cho trang phục công sở.\",\n    \"Vải chống nhăn, giữ form tốt.\",\n    \"Sản phẩm đa năng, dùng được nhiều cách.\"\n]\n\nnegative_feedbacks = [\n    \"Chất liệu vải kém, dễ rách sau vài lần mặc.\",\n    \"Size không chuẩn, quá chật hoặc quá rộng.\",\n    \"Màu sắc phai nhanh sau giặt.\",\n    \"Thiết kế lỗi thời, không hợp mốt.\",\n    \"Vải gây ngứa da, không thoải mái.\",\n    \"Đường may lỏng lẻo, dễ bung chỉ.\",\n    \"Không bền, hỏng sau thời gian ngắn.\",\n    \"Mùi hóa chất khó chịu từ vải mới.\",\n    \"Không thoáng khí, nóng bức khi mặc.\",\n    \"Giá cao nhưng chất lượng không xứng đáng.\",\n    \"Dễ nhăn, khó ủi phẳng.\",\n    \"Màu sắc không giống hình ảnh quảng cáo.\",\n    \"Phụ kiện kém chất lượng, dễ hỏng.\",\n    \"Không phù hợp với thời tiết địa phương.\",\n    \"Kiểu dáng không tôn dáng người mặc.\",\n    \"Vải mỏng, dễ lộ nội y.\",\n    \"Khó giặt sạch vết bẩn.\",\n    \"Size không đa dạng, khó chọn.\",\n    \"Sản phẩm lỗi, có vết bẩn từ nhà sản xuất.\",\n    \"Không giữ form sau giặt.\"\n]\n\nneutral_feedbacks = [\n    \"Sản phẩm bình thường, không có gì đặc biệt.\",\n    \"Chất lượng trung bình, dùng tạm được.\",\n    \"Thiết kế đơn giản, phù hợp cho mặc hàng ngày.\",\n    \"Màu sắc ổn, nhưng không nổi bật.\",\n    \"Size vừa phải, không quá chật hay rộng.\",\n    \"Vải khá, nhưng có thể tốt hơn.\",\n    \"Giá cả hợp lý cho chất lượng này.\",\n    \"Dùng được, nhưng không ấn tượng lắm.\",\n    \"Phù hợp cho mặc ở nhà.\",\n    \"Không tệ, nhưng cũng không xuất sắc.\",\n    \"Kiểu dáng cơ bản, dễ phối đồ.\",\n    \"Màu trung tính, an toàn.\",\n    \"Chất liệu ổn định, không thay đổi sau giặt.\",\n    \"Sản phẩm như mô tả, không bất ngờ.\",\n    \"Dùng tạm thời, có thể thay thế sau.\",\n    \"Không gây dị ứng, nhưng không thoải mái lắm.\",\n    \"Thiết kế phổ thông, ai cũng mặc được.\",\n    \"Giá rẻ, chất lượng tương xứng.\",\n    \"Không có vấn đề gì lớn.\",\n    \"Sản phẩm trung lập, tùy sở thích cá nhân.\"\n]\n\n# Gán ngẫu nhiên loại feedback cho mỗi transaction (interaction giữa customer và product)\n# Giả sử phân bố: 70% positive, 10% negative, 20% neutral để dữ liệu thiên về positive như thực tế\ntransactions['feedback_type'] = np.random.choice(['positive', 'negative', 'neutral'], size=len(transactions), p=[0.7, 0.1, 0.2])\n\n# Hàm để chọn ngẫu nhiên feedback dựa trên loại\ndef get_feedback(row):\n    if row['feedback_type'] == 'positive':\n        return random.choice(positive_feedbacks)\n    elif row['feedback_type'] == 'negative':\n        return random.choice(negative_feedbacks)\n    else:\n        return random.choice(neutral_feedbacks)\n\n# Thêm cột feedback\ntransactions['feedback'] = transactions.apply(get_feedback, axis=1)\n\n# Hàm để chọn ngẫu nhiên rating dựa trên loại\ndef get_rating(row):\n    if row['feedback_type'] == 'positive':\n        return round(random.uniform(3.5, 5.0), 1)\n    elif row['feedback_type'] == 'negative':\n        return round(random.uniform(0.0, 2.0), 1)\n    else:\n        return round(random.uniform(2.1, 3.4), 1)\n\n# Thêm cột rating\ntransactions['rating'] = transactions.apply(get_rating, axis=1)\n\n# In một số mẫu để kiểm tra\nprint(\"\\nMẫu dữ liệu transactions sau khi thêm feedback và rating:\")\nprint(transactions[['customer_id', 'article_id', 'feedback_type', 'feedback', 'rating']].head(10))\n\n# Vẽ biểu đồ pie cho tỷ lệ feedback_type\nfeedback_counts = transactions['feedback_type'].value_counts()\nplt.figure(figsize=(6, 6))\nfeedback_counts.plot(kind='pie', autopct='%1.1f%%', colors=['#4CAF50', '#F44336', '#FFC107'], startangle=90)\nplt.title('Feedback Type Ratio')\nplt.ylabel('')  # Ẩn nhãn y để biểu đồ sạch hơn\nplt.show()\n\n# Vẽ histogram cho phân bố rating\nplt.figure(figsize=(8, 6))\ntransactions['rating'].hist(bins=20, color='#2196F3', edgecolor='black')\nplt.title('Rating Distribution')\nplt.xlabel('Rating')\nplt.ylabel('Count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:49:49.625945Z","iopub.execute_input":"2025-08-14T11:49:49.626581Z","iopub.status.idle":"2025-08-14T11:56:56.033886Z","shell.execute_reply.started":"2025-08-14T11:49:49.626546Z","shell.execute_reply":"2025-08-14T11:56:56.032911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **ecofriendly**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Đọc dữ liệu articles.csv\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\n\n# Danh sách từ khóa eco-friendly trong mô tả sản phẩm\neco_keywords = [\n    'organic cotton', 'recycled polyester', 'relenya', 'recycled', 'organic',\n    'sustainable', 'eco-friendly', 'ethical', 'environmentally friendly',\n    'tencel', 'lyocell', 'hemp', 'bamboo', 'bio-based', 'recyclable',\n    'low-impact', 'vegan', 'GOTS', 'OEKO-TEX', 'Fair Trade', 'modal'\n]\n\n# Danh sách màu sắc eco-friendly mở rộng\nbase_eco_colors = [\n    'Beige', 'Light Beige', 'Grey', 'Light Grey', 'Dark Grey',\n    'Green', 'Light Green', 'Dark Green', 'Greenish Khaki',\n    'Brown', 'Yellowish Brown', 'Off White', 'White',\n    'Light Blue', 'Blue', 'Turquoise', 'Light Turquoise',\n    'Bronze/Copper'\n]\n\nadditional_eco_colors = ['Beige', 'Khaki', 'Olive Green', 'Brown', 'Off White', 'Taupe', 'Natural White']\neco_colors = base_eco_colors + additional_eco_colors\n\n# Hàm kiểm tra sản phẩm eco-friendly dựa trên detail_desc\ndef is_eco_friendly(description):\n    if pd.isna(description):\n        return False\n    return any(keyword in str(description).lower() for keyword in eco_keywords)\n\n# Đánh dấu sản phẩm eco-friendly dựa trên detail_desc\narticles['is_eco_desc'] = articles['detail_desc'].apply(is_eco_friendly)\n\n# Đánh dấu sản phẩm eco-friendly dựa trên màu sắc\narticles['is_eco_color'] = articles['colour_group_name'].isin(eco_colors)\n\n# Kết hợp: sản phẩm eco-friendly nếu thỏa mãn detail_desc HOẶC màu sắc\narticles['is_eco'] = articles['is_eco_desc'] | articles['is_eco_color']\n\n# Lọc các sản phẩm eco-friendly\neco_products = articles[articles['is_eco']]\nnon_eco_products = articles[~articles['is_eco']]\n\n# Tổng hợp số lượng sản phẩm\nprint(f\"Tổng số sản phẩm eco-friendly: {len(eco_products)}\")\nprint(f\"Số sản phẩm eco-friendly từ detail_desc: {articles['is_eco_desc'].sum()}\")\nprint(f\"Số sản phẩm eco-friendly từ màu sắc: {articles['is_eco_color'].sum()}\")\nprint(f\"Số sản phẩm không eco-friendly: {len(non_eco_products)}\")\n\n# Phân bố màu sắc trong sản phẩm eco-friendly\neco_color_dist = eco_products['colour_group_name'].value_counts(normalize=True).head(10)\nprint(\"\\nTop 10 màu sắc phổ biến trong sản phẩm eco-friendly (tỷ lệ):\")\nprint(eco_color_dist)\n\n# Phân bố giá trị màu sắc cảm nhận trong sản phẩm eco-friendly\neco_perceived_color_dist = eco_products['perceived_colour_value_name'].value_counts(normalize=True).head(10)\nprint(\"\\nTop 10 giá trị màu sắc cảm nhận trong sản phẩm eco-friendly (tỷ lệ):\")\nprint(eco_perceived_color_dist)\n\n# Vẽ biểu đồ pie cho tỷ lệ sản phẩm eco-friendly\neco_counts = pd.Series([len(eco_products), len(non_eco_products)], index=['Eco-Friendly', 'Non Eco-Friendly'])\nplt.figure(figsize=(6, 6))\neco_counts.plot(kind='pie', autopct='%1.1f%%', colors=['#4CAF50', '#F44336'], startangle=90)\nplt.title('Eco-Friendly Product Ratio')\nplt.ylabel('')  # Ẩn nhãn y để biểu đồ sạch hơn\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:57:09.196824Z","iopub.execute_input":"2025-08-14T11:57:09.197803Z","iopub.status.idle":"2025-08-14T11:57:11.171567Z","shell.execute_reply.started":"2025-08-14T11:57:09.197765Z","shell.execute_reply":"2025-08-14T11:57:11.170798Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**label cho ecofriendly**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Đọc dữ liệu articles.csv\narticles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\n\n# Danh sách từ khóa eco-friendly trong mô tả sản phẩm\neco_keywords = [\n    'organic cotton', 'recycled polyester', 'relenya', 'recycled', 'organic',\n    'sustainable', 'eco-friendly', 'ethical', 'environmentally friendly',\n    'tencel', 'lyocell', 'hemp', 'bamboo', 'bio-based', 'recyclable',\n    'low-impact', 'vegan', 'GOTS', 'OEKO-TEX', 'Fair Trade', 'modal'\n]\n\n# Danh sách màu sắc eco-friendly mở rộng\nbase_eco_colors = [\n    'Beige', 'Light Beige', 'Grey', 'Light Grey', 'Dark Grey',\n    'Green', 'Light Green', 'Dark Green', 'Greenish Khaki',\n    'Brown', 'Yellowish Brown', 'Off White', 'White',\n    'Light Blue', 'Blue', 'Turquoise', 'Light Turquoise',\n    'Bronze/Copper'\n]\nadditional_eco_colors = ['Beige', 'Khaki', 'Olive Green', 'Brown', 'Off White', 'Taupe', 'Natural White']\neco_colors = base_eco_colors + additional_eco_colors\n\n# Hàm kiểm tra sản phẩm eco-friendly dựa trên detail_desc\ndef is_eco_friendly(description):\n    if pd.isna(description):\n        return False\n    return any(keyword in str(description).lower() for keyword in eco_keywords)\n\n# Đánh dấu sản phẩm eco-friendly dựa trên detail_desc\narticles['is_eco_desc'] = articles['detail_desc'].apply(is_eco_friendly)\n\n# Đánh dấu sản phẩm eco-friendly dựa trên màu sắc\narticles['is_eco_color'] = articles['colour_group_name'].isin(eco_colors)\n\n# Kết hợp: sản phẩm eco-friendly nếu thỏa mãn detail_desc HOẶC màu sắc\narticles['is_eco'] = articles['is_eco_desc'] | articles['is_eco_color']\n\n# Thêm cột label: True nếu is_eco là True (eco-friendly), False nếu không\narticles['eco_label'] = articles['is_eco']  # Giữ nguyên boolean True/False\n\n# In một số mẫu để kiểm tra\nprint(\"\\nMẫu dữ liệu articles sau khi thêm label:\")\nprint(articles[['article_id', 'is_eco', 'eco_label']].head(10))\n\n# Lưu dataframe articles với cột mới vào file CSV mới\n# Sử dụng đường dẫn /kaggle/working/ để lưu trong Kaggle (có thể tải xuống sau)\narticles.to_csv('/kaggle/working/articles_with_eco_label.csv', index=False)\n\nprint(\"\\nFile đã được lưu tại /kaggle/working/articles_with_eco_label.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:57:58.192312Z","iopub.execute_input":"2025-08-14T11:57:58.192631Z","iopub.status.idle":"2025-08-14T11:58:01.813767Z","shell.execute_reply.started":"2025-08-14T11:57:58.192608Z","shell.execute_reply":"2025-08-14T11:58:01.812581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **CHIA 80/20**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport matplotlib.pyplot as plt\n\n# Merge tất cả để có dataframe đầy đủ\nmerged_df = pd.merge(transactions, articles[['article_id', 'eco_label']], on='article_id', how='left')\n\n# labels có index là customer_id, cần reset_index nếu chưa\nif 'customer_id' not in labels.columns:\n    labels = labels.reset_index()\n\nmerged_df = pd.merge(merged_df, labels[['customer_id', 'final_gender']], on='customer_id', how='left')\n\n# Chuyển t_dat sang datetime nếu chưa\nmerged_df['t_dat'] = pd.to_datetime(merged_df['t_dat'])\n\n# Sort theo t_dat tăng dần\nmerged_df = merged_df.sort_values(by='t_dat').reset_index(drop=True)\n\n# Tính điểm cắt cho 80% train, 20% test\nsplit_index = int(len(merged_df) * 0.8)\ntrain_data = merged_df.iloc[:split_index]\ntest_data = merged_df.iloc[split_index:]\n\n# Lưu vào file\ntrain_data.to_csv('/kaggle/working/train_data_new.csv', index=False)\ntest_data.to_csv('/kaggle/working/test_data_new.csv', index=False)\n\n# In cấu trúc\nprint(\"Cấu trúc của train_data:\")\nprint(\"Shape:\", train_data.shape)\nprint(\"Columns:\", train_data.columns.tolist())\nprint(\"Head:\\n\", train_data.head())\n\nprint(\"\\nCấu trúc của test_data:\")\nprint(\"Shape:\", test_data.shape)\nprint(\"Columns:\", test_data.columns.tolist())\nprint(\"Head:\\n\", test_data.head())\n\n# Vẽ biểu đồ cột cho số lượng dữ liệu train và test\ndata_sizes = pd.Series([len(train_data), len(test_data)], index=['Train', 'Test'])\nplt.figure(figsize=(6, 4))\ndata_sizes.plot(kind='bar', color=['#1f77b4', '#ff7f0e'])\nplt.title('Number of Transactions in Train and Test Data')\nplt.xlabel('Dataset')\nplt.ylabel('Count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T11:58:39.168760Z","iopub.execute_input":"2025-08-14T11:58:39.169080Z","iopub.status.idle":"2025-08-14T12:05:01.699606Z","shell.execute_reply.started":"2025-08-14T11:58:39.169055Z","shell.execute_reply":"2025-08-14T12:05:01.698566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **EfficientNetB**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Đọc file train & test\ntrain_data = pd.read_csv('/kaggle/input/chiadulieu8020new/train_data_new.csv')\ntest_data = pd.read_csv('/kaggle/input/chiadulieu8020new/test_data_new.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T12:29:13.699326Z","iopub.execute_input":"2025-08-14T12:29:13.699652Z","iopub.status.idle":"2025-08-14T12:32:19.329736Z","shell.execute_reply.started":"2025-08-14T12:29:13.699630Z","shell.execute_reply":"2025-08-14T12:32:19.328815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import classification_report, confusion_matrix\n\n\n# Đọc dữ liệu Kaggle\ntrain_data = pd.read_csv(\"/kaggle/input/chiadulieu8020new/train_data_new.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/chiadulieu8020new/test_data_new.csv\")\ntrain_sample_size = 5000\ntest_sample_size = 5000\ntrain_data = train_data.sample(n=train_sample_size, random_state=42)\ntest_data = test_data.sample(n=test_sample_size, random_state=42)\n\ndef preprocess(df):\n    df = df.copy()\n    # 1) t_dat -> đặc trưng số\n    df[\"t_dat\"] = pd.to_datetime(df[\"t_dat\"], errors=\"coerce\")\n    df[\"month\"] = df[\"t_dat\"].dt.month\n    df[\"dayofweek\"] = df[\"t_dat\"].dt.dayofweek\n    df[\"year\"] = df[\"t_dat\"].dt.year  # Added year feature\n\n    # 2) feedback_type -> 0/1\n    df[\"feedback_type\"] = df[\"feedback_type\"].map({\"positive\": 1, \"negative\": 0}).fillna(0)\n\n    # 3) eco_label bool/chuỗi -> 0/1\n    if df[\"eco_label\"].dtype == bool:\n        df[\"eco_label\"] = df[\"eco_label\"].astype(int)\n    else:\n        df[\"eco_label\"] = df[\"eco_label\"].map({\"True\": 1, \"False\": 0}).fillna(0).astype(int)\n\n    # 4) Bỏ các cột không dùng/không phải số\n    df = df.drop(columns=[\"t_dat\", \"customer_id\", \"feedback\"], errors=\"ignore\")\n\n    # (tuỳ chọn) đảm bảo article_id là số\n    df[\"article_id\"] = pd.to_numeric(df[\"article_id\"], errors=\"coerce\")\n\n    # Điền thiếu\n    df = df.fillna(df.median(numeric_only=True), inplace=False)  # Use median for numerical columns\n    return df\n\ntrain_pp = preprocess(train_data)\ntest_pp = preprocess(test_data)\n\n# Tạo X, y chỉ từ cột số\nfeature_columns = [c for c in train_pp.columns if c != \"final_gender\"]\nnon_num = train_pp[feature_columns].select_dtypes(exclude=[np.number]).columns.tolist()\nif non_num:\n    print(f\"Warning: Non-numeric columns found: {non_num}\")\n\nX_train = train_pp[feature_columns].to_numpy(dtype=np.float32)\nX_test = test_pp[feature_columns].to_numpy(dtype=np.float32)\ny_train = train_pp[\"final_gender\"]\ny_test = test_pp[\"final_gender\"]\n\n# Chuẩn hóa dữ liệu\nscaler = MinMaxScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)\n\n# Chuyển thành ma trận vuông để dùng EfficientNet\nimg_size = int(np.ceil(np.sqrt(X_train.shape[1])))\nimg_size = max(img_size, 32)\n\npad_len = img_size**2 - X_train.shape[1]\nif pad_len > 0:\n    X_train_padded = np.pad(X_train, ((0, 0), (0, pad_len)), 'constant')\n    X_test_padded = np.pad(X_test, ((0, 0), (0, pad_len)), 'constant')\nelse:\n    X_train_padded, X_test_padded = X_train, X_test\n\n# Reshape thành ảnh\nX_train_images = X_train_padded.reshape(-1, img_size, img_size, 1)\nX_test_images = X_test_padded.reshape(-1, img_size, img_size, 1)\nprint(\"✅ Shape dữ liệu ảnh train:\", X_train_images.shape)\nprint(\"✅ Shape dữ liệu ảnh test:\", X_test_images.shape)\n\n# Mã hóa nhãn\nencoder = LabelEncoder()\nall_labels = pd.concat([train_data[\"final_gender\"], test_data[\"final_gender\"]]).unique()\nencoder.fit(all_labels)\ny_train_encoded = encoder.transform(y_train)\ny_test_encoded = encoder.transform(y_test)\ny_train_categorical = to_categorical(y_train_encoded)\ny_test_categorical = to_categorical(y_test_encoded)\n\n# Xây dựng mô hình\ninputs = layers.Input(shape=(img_size, img_size, 1))\nx = layers.Conv2D(3, (3, 3), padding='same', activation='relu')(inputs)\nbase_model = EfficientNetB0(weights=None, include_top=False, input_tensor=x)\nx = layers.GlobalAveragePooling2D()(base_model.output)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(0.3)(x)\noutputs = layers.Dense(y_train_categorical.shape[1], activation='softmax')(x)\n\nmodel = models.Model(inputs, outputs)\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Callbacks\nearly_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\nlr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3)\n\n# Huấn luyện mô hình\nhistory = model.fit(\n    X_train_images, y_train_categorical,\n    validation_data=(X_test_images, y_test_categorical),\n    epochs=10,  # Increased epochs, relying on early stopping\n    batch_size=64,\n    callbacks=[early_stopping, lr_scheduler],\n    verbose=1\n)\n\n# Đánh giá mô hình\nval_loss, val_acc = model.evaluate(X_test_images, y_test_categorical, verbose=1)\nprint(f\"Validation Loss: {val_loss:.4f}\")\nprint(f\"Validation Accuracy: {val_acc:.4f}\")\n\n# Báo cáo phân loại\nfrom sklearn.metrics import classification_report\ny_pred = model.predict(X_test_images)\ny_pred_classes = np.argmax(y_pred, axis=1)\nprint(classification_report(y_test_encoded, y_pred_classes, target_names=encoder.classes_))\n\n# Vẽ biểu đồ accuracy và loss\nplt.figure(figsize=(12, 4))\n\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Train Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Model Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(loc='upper left')\n\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Train Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Model Loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(loc='upper left')\n\nplt.show()\n\n# Vẽ confusion matrix\ncm = confusion_matrix(y_test_encoded, y_pred_classes)\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=encoder.classes_, yticklabels=encoder.classes_)\nplt.title('Confusion Matrix')\nplt.xlabel('Predicted')\nplt.ylabel('True')\nplt.show()\n\n# Vẽ biểu đồ precision, recall, f1-score\nreport = classification_report(y_test_encoded, y_pred_classes, target_names=encoder.classes_, output_dict=True)\nmetrics_df = pd.DataFrame(report).transpose().drop(['support'], axis=1)[:-3]  # Loại bỏ accuracy, macro avg, weighted avg\nmetrics_df.plot(kind='bar', figsize=(10, 6))\nplt.title('Precision, Recall, F1-Score per Class')\nplt.ylabel('Score')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T12:33:16.402975Z","iopub.execute_input":"2025-08-14T12:33:16.403393Z","iopub.status.idle":"2025-08-14T12:41:57.907884Z","shell.execute_reply.started":"2025-08-14T12:33:16.403366Z","shell.execute_reply":"2025-08-14T12:41:57.906889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**CF**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nmerged = pd.read_csv('/kaggle/input/traintestlan2new/train_kmeans_kmedoids.csv')\nprint(merged.head())\nprint(merged.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:50:11.592579Z","iopub.execute_input":"2025-08-14T13:50:11.595904Z","iopub.status.idle":"2025-08-14T13:53:23.718013Z","shell.execute_reply.started":"2025-08-14T13:50:11.595776Z","shell.execute_reply":"2025-08-14T13:53:23.716844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom scipy.sparse import csr_matrix\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n# Bước 2: Kiểm tra và làm sạch dữ liệu\nrequired_columns = ['customer_id', 'article_id', 'rating', 'season', 'product_group_name']\noptional_columns = ['user_season_cluster', 'final_gender', 'eco_label']\nall_columns = required_columns + [col for col in optional_columns if col in merged.columns]\nprint(\"Số hàng trong merged:\", len(merged))\nprint(\"Sample merged data:\\n\", merged[all_columns].head())\nif len(merged) == 0:\n    raise ValueError(\"DataFrame merged is empty. Check data loading or merging process.\")\nif not all(col in merged.columns for col in required_columns):\n    raise ValueError(f\"Missing required columns. Required: {required_columns}, Found: {merged.columns}\")\nif merged[required_columns].isna().any().any():\n    print(\"Warning: NaN values found in required columns. Filling with defaults.\")\n    merged['rating'] = merged['rating'].fillna(0)\n    merged['season'] = merged['season'].fillna(0)  # Sửa từ 'unknown' thành 0 cho consistency\n# Bước 3: Lọc top 1000 người dùng\nuser_freq = merged['customer_id'].value_counts()\ntop_users = user_freq.head(5000).index\nmerged_subset = merged[merged['customer_id'].isin(top_users)]\ntrain_data = merged_subset  # Không chia train/test vì chỉ huấn luyện\nprint(\"Số hàng trong train_data:\", len(train_data))\nprint(\"Số user trong train_data:\", train_data['customer_id'].nunique())\nprint(\"Số article trong train_data:\", train_data['article_id'].nunique())\nif len(merged_subset) == 0:\n    raise ValueError(\"merged_subset is empty. Check customer_id filtering or data integrity.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:54:43.651293Z","iopub.execute_input":"2025-08-14T13:54:43.652218Z","iopub.status.idle":"2025-08-14T13:55:04.148400Z","shell.execute_reply.started":"2025-08-14T13:54:43.652176Z","shell.execute_reply":"2025-08-14T13:55:04.146835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 4: Tạo ma trận user-item\nuser_item_matrix_train = train_data.pivot_table(\n    index='customer_id',\n    columns='article_id',\n    values='rating',\n    aggfunc='sum'\n).fillna(0)\n\nuser_item_values_train = csr_matrix(user_item_matrix_train.values)\nprint(\"Ma trận user-item (train):\", user_item_values_train.shape)  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:56:11.456327Z","iopub.execute_input":"2025-08-14T13:56:11.456752Z","iopub.status.idle":"2025-08-14T13:56:38.038531Z","shell.execute_reply.started":"2025-08-14T13:56:11.456723Z","shell.execute_reply":"2025-08-14T13:56:38.037482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tính độ thưa thớt\nsparsity = 1 - np.count_nonzero(user_item_matrix_train.values) / user_item_matrix_train.values.size\nprint(\"Độ thưa thớt của user_item_matrix_train:\", sparsity)\n\nif user_item_matrix_train.shape[0] == 0 or user_item_matrix_train.shape[1] == 0:\n    raise ValueError(\"user_item_matrix_train is empty.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:57:00.902077Z","iopub.execute_input":"2025-08-14T13:57:00.902463Z","iopub.status.idle":"2025-08-14T13:57:01.965666Z","shell.execute_reply.started":"2025-08-14T13:57:00.902436Z","shell.execute_reply":"2025-08-14T13:57:01.964381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 5: Tính similarity user-user\nuser_similarity_train = cosine_similarity(user_item_values_train)\nprint(\"Shape ma trận similarity (train):\", user_similarity_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:57:22.420019Z","iopub.execute_input":"2025-08-14T13:57:22.420365Z","iopub.status.idle":"2025-08-14T13:57:23.561229Z","shell.execute_reply.started":"2025-08-14T13:57:22.420339Z","shell.execute_reply":"2025-08-14T13:57:23.559958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 6: Dự đoán ratings\nratings_train = user_item_values_train.toarray()\nnum_users, num_items = ratings_train.shape\npredicted_ratings = np.dot(user_similarity_train, ratings_train) / np.sum(np.abs(user_similarity_train), axis=1, keepdims=True)\npredicted_ratings[np.isnan(predicted_ratings)] = 0\nprint(\"Dự đoán ma trận ratings hoàn tất:\", predicted_ratings.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T13:57:52.761252Z","iopub.execute_input":"2025-08-14T13:57:52.762345Z","iopub.status.idle":"2025-08-14T13:58:57.143035Z","shell.execute_reply.started":"2025-08-14T13:57:52.762309Z","shell.execute_reply":"2025-08-14T13:58:57.142087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 7: Tạo user_fav_cluster\nuser_fav_cluster = {}\ncurrent_season = 0\nfor user_id in user_item_matrix_train.index:\n    bought_items = user_item_matrix_train.loc[user_id]\n    bought_items = bought_items[bought_items > 0].index.tolist()\n    if bought_items:\n        clusters = train_data[train_data['article_id'].isin(bought_items)]['season']\n        if not clusters.empty:\n            user_fav_cluster[user_id] = clusters.mode().iloc[0]\n        else:\n            user_fav_cluster[user_id] = current_season\n    else:\n        user_cluster = train_data[train_data['customer_id'] == user_id]['user_season_cluster']\n        if not user_cluster.empty:\n            user_fav_cluster[user_id] = user_cluster.mode().iloc[0]\n        else:\n            user_fav_cluster[user_id] = current_season\n# Debug: Kiểm tra user_fav_cluster\nprint(\"Số user có fav_cluster:\", len(user_fav_cluster))\nprint(\"Sample user_fav_cluster:\", list(user_fav_cluster.items())[:5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:00:09.042293Z","iopub.execute_input":"2025-08-14T14:00:09.042730Z","iopub.status.idle":"2025-08-14T14:04:49.925947Z","shell.execute_reply.started":"2025-08-14T14:00:09.042701Z","shell.execute_reply":"2025-08-14T14:04:49.924913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 8: Tạo mapping\nitem_cluster_map = dict(zip(train_data['article_id'], train_data['season']))\nitem_gender_map = dict(zip(train_data['article_id'], train_data['final_gender']))\nitem_eco_map = dict(zip(train_data['article_id'], train_data['eco_label']))\nitem_product_group_map = dict(zip(train_data['article_id'], train_data['product_group_name']))\n\nmissing_items = set(user_item_matrix_train.columns) - set(item_cluster_map.keys())\nif missing_items:\n    print(f\"Warning: {len(missing_items)} article_id(s) missing seasonr\")\n    for item_id in missing_items:\n        item_cluster_map[item_id] = current_season\nprint(\"Số article_id có season_cluster:\", len(item_cluster_map))\nprint(\"Sample item_cluster_map:\", list(item_cluster_map.items())[:5])\nprint(\"Số article_id có product_group_name:\", len(item_product_group_map))\nprint(\"Sample item_product_group_map:\", list(item_product_group_map.items())[:5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:05:23.194636Z","iopub.execute_input":"2025-08-14T14:05:23.195812Z","iopub.status.idle":"2025-08-14T14:05:25.323882Z","shell.execute_reply.started":"2025-08-14T14:05:23.195775Z","shell.execute_reply":"2025-08-14T14:05:25.322653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 9: Tạo recommendation\nrecommendations = {}\nskipped_users = 0\nempty_filtered_items = 0\nuser_gender_map = {}\nfor user_idx, user_id in enumerate(user_item_matrix_train.index):\n    fav_cluster = user_fav_cluster.get(user_id, current_season)\n    if fav_cluster is None:\n        skipped_users += 1\n        continue\n    user_gender = user_gender_map.get(user_id, None)\n    preds = predicted_ratings[user_idx, :]\n    item_ids = user_item_matrix_train.columns\n    item_mask = np.array([\n        item_cluster_map.get(iid, current_season) == fav_cluster\n        for iid in item_ids\n    ])\n    filtered_indices = np.where(item_mask)[0]\n    pred_items_filtered = [\n        (item_ids[i], preds[i] * (1.2 if item_eco_map.get(item_ids[i], False) else 1.0))\n        for i in filtered_indices\n    ]\n    if not pred_items_filtered:\n        empty_filtered_items += 1\n        popular_items = train_data[\n            (train_data['season'] == fav_cluster) &\n            (train_data['eco_label'] == True)\n        ]['article_id'].value_counts().head(5).index\n        if not popular_items.empty:\n            pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n        else:\n            popular_items = train_data[\n                (train_data['season'] == fav_cluster)\n            ]['article_id'].value_counts().head(5).index\n            if not popular_items.empty:\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n            else:\n                popular_items = train_data['article_id'].value_counts().head(5).index\n                pred_items_filtered = [(iid, 0.0) for iid in popular_items]\n    pred_items_filtered.sort(key=lambda x: x[1], reverse=True)\n    recommendations[user_id] = pred_items_filtered[:5]\nprint(\"Tạo xong recommendation hybrid cho\", len(recommendations), \"users.\")\nprint(\"Số user bị bỏ qua (fav_cluster is None):\", skipped_users)\nprint(\"Số user có pred_items_filtered rỗng:\", empty_filtered_items)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:06:12.503043Z","iopub.execute_input":"2025-08-14T14:06:12.503707Z","iopub.status.idle":"2025-08-14T14:13:48.900578Z","shell.execute_reply.started":"2025-08-14T14:06:12.503647Z","shell.execute_reply":"2025-08-14T14:13:48.899535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.metrics import mean_squared_error\ntest_data = pd.read_csv('/kaggle/input/traintestlan2/test_kmeans_kmedoids.csv')\nif len(test_data) > 5000:\n    test_data = test_data[test_data['customer_id'].isin(top_users)].sample(n=5000, random_state=42)\nelse:\n    print(f\"Warning: Số hàng trong test_data ({len(test_data)}) nhỏ hơn 5,000. Sử dụng toàn bộ dữ liệu.\")\nrequired_columns = ['customer_id', 'article_id', 'rating', 'season', 'product_group_name']\noptional_columns = ['user_season_cluster', 'final_gender', 'eco_label']\nall_columns = required_columns + [col for col in optional_columns if col in test_data.columns]\nuser_item_matrix_test = test_data.pivot_table(\n    index='customer_id',\n    columns='article_id',\n    values='rating',\n    aggfunc='sum'\n).fillna(0)\ncommon_users = user_item_matrix_train.index.intersection(user_item_matrix_test.index)\ncommon_items = user_item_matrix_train.columns.intersection(user_item_matrix_test.columns)\nprint(\"Shape của predicted_ratings:\", predicted_ratings.shape)\nprint(\"Số common_users:\", len(common_users))\nprint(\"Số common_items:\", len(common_items))\nprint(\"Shape của user_item_matrix_train:\", user_item_matrix_train.shape)\nprint(\"Shape của user_item_matrix_test:\", user_item_matrix_test.shape)\nif len(common_users) > 0 and len(common_items) > 0:\n    # Lấy true ratings và predicted ratings\n    true_ratings = user_item_matrix_test.loc[common_users, common_items].values\n    user_indices = user_item_matrix_train.index.get_indexer(common_users)\n    item_indices = user_item_matrix_train.columns.get_indexer(common_items)\n    # Kiểm tra chỉ số hợp lệ riêng lẻ\n    valid_user_mask = user_indices != -1\n    valid_item_mask = item_indices != -1\n    if not np.all(valid_user_mask) or not np.all(valid_item_mask):\n        print(\"Warning: Some users or items not found in user_item_matrix_train.\")\n        valid_indices = valid_user_mask & (valid_item_mask[:len(valid_user_mask)])\n        common_users = common_users[valid_indices]\n        common_items = common_items[valid_indices[:len(common_items)]]\n        user_indices = user_indices[valid_indices]\n        item_indices = item_indices[valid_indices[:len(item_indices)]]\n        true_ratings = user_item_matrix_test.loc[common_users, common_items].values\n    if len(user_indices) == 0 or len(item_indices) == 0:\n        print(\"Error: No valid indices after filtering. Cannot compute RMSE.\")\n    else:\n        try:\n            pred_ratings = predicted_ratings[user_indices][:, item_indices]\n            print(\"Shape của true_ratings:\", true_ratings.shape)\n            print(\"Shape của pred_ratings:\", pred_ratings.shape)      \n            if true_ratings.shape != pred_ratings.shape:\n                print(\"Error: Shape mismatch between true_ratings and pred_ratings.\")\n            else:\n              mask = true_ratings > 0\n              if np.sum(mask) > 0:\n                    rmse = np.sqrt(mean_squared_error(true_ratings[mask], pred_ratings[mask]))\n                    print(\"RMSE on test set:\", rmse)\n              else:\n                    print(\"Warning: No non-zero true ratings to compute RMSE.\")\n        except IndexError as e:\n            print(f\"IndexError: {e}\")\n            print(\"Cannot compute RMSE due to indexing issues.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:15:19.309654Z","iopub.execute_input":"2025-08-14T14:15:19.310051Z","iopub.status.idle":"2025-08-14T14:16:07.290205Z","shell.execute_reply.started":"2025-08-14T14:15:19.310027Z","shell.execute_reply":"2025-08-14T14:16:07.289127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_data.head())\nprint(test_data.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:16:51.563585Z","iopub.execute_input":"2025-08-14T14:16:51.564719Z","iopub.status.idle":"2025-08-14T14:16:51.575065Z","shell.execute_reply.started":"2025-08-14T14:16:51.564654Z","shell.execute_reply":"2025-08-14T14:16:51.573973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\n\nif np.sum(mask) > 0:\n    mae = mean_absolute_error(true_ratings[mask], pred_ratings[mask])\n    print(\"MAE on test set:\", mae)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:17:00.915061Z","iopub.execute_input":"2025-08-14T14:17:00.915378Z","iopub.status.idle":"2025-08-14T14:17:00.970478Z","shell.execute_reply.started":"2025-08-14T14:17:00.915358Z","shell.execute_reply":"2025-08-14T14:17:00.969300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_recommendations(recommendations, test_data, k=5):\n    precision = recall = f1 = ap_sum = total = 0\n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            true_items = test_data[test_data['customer_id'] == user_id]['article_id'].tolist()\n            if not rec_items or not true_items:\n                continue\n            hits = len(set(rec_items).intersection(true_items))\n            if hits > 0:\n                print(f\"User {user_id}: rec_items={rec_items}, true_items={true_items}, hits={hits}\")\n                precision += hits / k\n                recall += hits / len(true_items)\n                ap = 0\n                relevant_count = 0\n                for i, item in enumerate(rec_items[:k], 1):\n                    if item in true_items:\n                        relevant_count += 1\n                        ap += relevant_count / i\n                ap = ap / min(len(true_items), k)\n                ap_sum += ap\n                total += 1\n    precision = precision / total if total > 0 else 0\n    recall = recall / total if total > 0 else 0\n    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n    map_score = ap_sum / total if total > 0 else 0\n    print(f\"Debug - Total users evaluated: {total}\")\n    return {'Precision@5': precision, 'Recall@5': recall, 'F1-Score@5': f1, 'MAP@5': map_score}\nprint('evaluate_recommendations',evaluate_recommendations(recommendations, test_data, k=5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:17:42.125405Z","iopub.execute_input":"2025-08-14T14:17:42.125775Z","iopub.status.idle":"2025-08-14T14:17:48.940911Z","shell.execute_reply.started":"2025-08-14T14:17:42.125751Z","shell.execute_reply":"2025-08-14T14:17:48.939946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recommendation1(recommendations, test_data, k=5):\n    precision = 0\n    recall = 0\n    f1 = 0\n    ap_sum = 0\n    total = 0    \n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            true_items = test_data[test_data['customer_id'] == user_id]['article_id'].tolist()\n            # Tính Precision@5\n            hits = len(set(rec_items).intersection(true_items))\n            precision += hits / k if k > 0 else 0\n            # Tính Recall@5\n            recall += hits / len(true_items) if len(true_items) > 0 else 0\n            # Tính AP@5 (Average Precision)\n            ap = 0\n            relevant_count = 0\n            for i, item in enumerate(rec_items[:k], 1):\n                if item in true_items:\n                    relevant_count += 1\n                    ap += relevant_count / i\n            ap = ap / min(len(true_items), k) if len(true_items) > 0 else 0\n            ap_sum += ap\n            total += 1\n    precision = precision / total if total > 0 else 0\n    recall = recall / total if total > 0 else 0\n    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n    map_score = ap_sum / total if total > 0 else 0\n    \n    return {\n        'Precision@5': precision,\n        'Recall@5': recall,\n        'F1-Score@5': f1,\n        'MAP@5': map_score\n    }\neval_metrics = evaluate_recommendations(recommendations, test_data, k=5)\nprint(\"Evaluation metrics:\")\nfor metric, value in eval_metrics.items():\n    print(f\"{metric}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:18:22.094876Z","iopub.execute_input":"2025-08-14T14:18:22.095513Z","iopub.status.idle":"2025-08-14T14:18:28.893475Z","shell.execute_reply.started":"2025-08-14T14:18:22.095487Z","shell.execute_reply":"2025-08-14T14:18:28.892482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import ndcg_score\ndef calculate_ndcg(recommendations, test_data, k=5):\n    ndcg_sum = 0\n    total = 0\n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]  # Top k gợi ý\n            # Lấy true_items và true_ratings tương ứng\n            user_data = test_data[test_data['customer_id'] == user_id]\n            true_items = user_data['article_id'].tolist()\n            true_ratings = user_data['rating'].tolist()  # Sử dụng cột 'ratings'\n            \n            if not rec_items or not true_items or not true_ratings:\n                continue\n                \n            # Tạo mảng relevance dựa trên ratings của true_items\n            relevance_scores = []\n            for item in rec_items:\n                if item in true_items:\n                    idx = true_items.index(item)\n                    relevance_scores.append(true_ratings[idx] if idx < len(true_ratings) else 0)\n                else:\n                    relevance_scores.append(0)  # Nếu item không trong true_items, gán 0\n                    \n            # Đảm bảo độ dài khớp\n            if len(relevance_scores) != k:\n                relevance_scores = relevance_scores[:k] + [0] * (k - len(relevance_scores))\n                \n            if sum(relevance_scores) > 0:  # Chỉ tính nếu có ratings > 0\n                ndcg_sum += ndcg_score([relevance_scores], [relevance_scores])  # So sánh với chính nó để kiểm tra\n                total += 1\n    return ndcg_sum / total if total > 0 else 0\n\n# Tính và in NDCG@5\nndcg = calculate_ndcg(recommendations, test_data, k=5)\nprint(f\"NDCG@5: {ndcg}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:19:39.155006Z","iopub.execute_input":"2025-08-14T14:19:39.155337Z","iopub.status.idle":"2025-08-14T14:19:45.977378Z","shell.execute_reply.started":"2025-08-14T14:19:39.155316Z","shell.execute_reply":"2025-08-14T14:19:45.976354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import ndcg_score\n\ndef calculate_ndcg1(recommendations, test_data, k=5):\n    ndcg_sum = 0\n    total = 0\n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            user_data = test_data[test_data['customer_id'] == user_id]\n            true_items = user_data['article_id'].tolist()\n            true_ratings = user_data['rating'].tolist()\n\n            if not rec_items or not true_items or not true_ratings:\n                continue\n\n            relevance_scores = []\n            ideal_scores = []\n            for item in rec_items:\n                if item in true_items:\n                    idx = true_items.index(item)\n                    relevance_scores.append(true_ratings[idx])\n                else:\n                    relevance_scores.append(0)\n\n            # Ideal ranking: sort true ratings in descending order\n            ideal_scores = sorted(relevance_scores, reverse=True)\n\n            if sum(ideal_scores) > 0:\n                ndcg_sum += ndcg_score([ideal_scores], [relevance_scores])\n                total += 1\n\n    return ndcg_sum / total if total > 0 else 0\nndcg1 = calculate_ndcg1(recommendations, test_data, k=5)\nprint(f\"NDCG@5: {ndcg1}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:28:27.959631Z","iopub.execute_input":"2025-08-14T14:28:27.959974Z","iopub.status.idle":"2025-08-14T14:28:34.676828Z","shell.execute_reply.started":"2025-08-14T14:28:27.959944Z","shell.execute_reply":"2025-08-14T14:28:34.675873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_hit_rate(recommendations, test_data, k=5):\n    hits = 0\n    total_users = 0\n    for user_id in recommendations:\n        if user_id in test_data['customer_id'].values:\n            rec_items = [r[0] for r in recommendations[user_id][:k]]\n            true_items = test_data[test_data['customer_id'] == user_id]['article_id'].tolist()\n            if not rec_items or not true_items:\n                continue\n            if any(item in true_items for item in rec_items):\n                hits += 1\n            total_users += 1\n    return hits / total_users if total_users > 0 else 0\nhit_rate = calculate_hit_rate(recommendations, test_data, k=5)\nprint(f\"Hit Rate@5: {hit_rate}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:20:37.415092Z","iopub.execute_input":"2025-08-14T14:20:37.415510Z","iopub.status.idle":"2025-08-14T14:20:44.043128Z","shell.execute_reply.started":"2025-08-14T14:20:37.415486Z","shell.execute_reply":"2025-08-14T14:20:44.042092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_coverage(recommendations, test_data):\n    all_rec_items = set()\n    for user_id in recommendations:\n        all_rec_items.update([r[0] for r in recommendations[user_id]])\n    total_items = set(test_data['article_id'].unique())\n    return len(all_rec_items) / len(total_items) if len(total_items) > 0 else 0\ncoverage = calculate_coverage(recommendations, test_data)\nprint(f\"Coverage: {coverage}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:21:12.852400Z","iopub.execute_input":"2025-08-14T14:21:12.852851Z","iopub.status.idle":"2025-08-14T14:21:12.874483Z","shell.execute_reply.started":"2025-08-14T14:21:12.852818Z","shell.execute_reply":"2025-08-14T14:21:12.873517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**CBF** cách 1","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.neighbors import NearestNeighbors\n\n# 1. Chuẩn bị dữ liệu features\ntrain_data = train_data.copy()\ntrain_data['features'] = (\n    train_data['product_group_name'].fillna('') + ' ' +\n    train_data['final_gender'].fillna('') + ' ' +\n    train_data['season'].fillna('') + ' ' +\n    train_data['feedback'].fillna('')\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:21:44.243477Z","iopub.execute_input":"2025-08-14T14:21:44.243864Z","iopub.status.idle":"2025-08-14T14:21:46.421600Z","shell.execute_reply.started":"2025-08-14T14:21:44.243837Z","shell.execute_reply":"2025-08-14T14:21:46.420252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.neighbors import NearestNeighbors\n\n# TF-IDF\nvectorizer = TfidfVectorizer(stop_words='english', max_features=2000)\ntfidf_matrix = vectorizer.fit_transform(train_data['features'])\n\n# Mapping article_id -> index\narticle_id_to_idx = {aid: idx for idx, aid in enumerate(train_data['article_id'])}\n\n# Nearest Neighbors (tìm top N nhanh)\nnn_model = NearestNeighbors(metric='cosine', algorithm='brute', n_jobs=-1)\nnn_model.fit(tfidf_matrix)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:22:16.993296Z","iopub.execute_input":"2025-08-14T14:22:16.993821Z","iopub.status.idle":"2025-08-14T14:22:41.791065Z","shell.execute_reply.started":"2025-08-14T14:22:16.993780Z","shell.execute_reply":"2025-08-14T14:22:41.789868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cbf_cache = {}\n\ndef get_cbf_recommendations_fast(article_id, top_n=10):\n    if article_id in cbf_cache:\n        return cbf_cache[article_id]\n    if article_id not in article_id_to_idx:\n        return []\n\n    idx = article_id_to_idx[article_id]\n    distances, indices = nn_model.kneighbors(tfidf_matrix[idx], n_neighbors=top_n+1)  # +1 vì có chính nó\n    recs = [(train_data.iloc[i]['article_id'], 1 - distances[0][j])\n            for j, i in enumerate(indices[0]) if i != idx]\n\n    cbf_cache[article_id] = recs\n    return recs\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:31:22.000735Z","iopub.execute_input":"2025-08-14T14:31:22.001234Z","iopub.status.idle":"2025-08-14T14:31:22.009976Z","shell.execute_reply.started":"2025-08-14T14:31:22.001203Z","shell.execute_reply":"2025-08-14T14:31:22.008632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_precision_recall(customer_id, k=5, recommend_func=None):\n    true_items = set(test_data[test_data['customer_id'] == customer_id]['article_id'])\n    if not true_items:\n        return None, None\n\n    start_items = train_data[train_data['customer_id'] == customer_id]['article_id'].tolist()\n    if not start_items:\n        return None, None\n    start_item = start_items[0]\n\n    recs = recommend_func(start_item, top_n=k)\n    recommended_items = set(aid for aid, score in recs)  # chỉ lấy article_id\n    if not recommended_items:\n        return None, None\n\n    precision = len(recommended_items & true_items) / k\n    recall = len(recommended_items & true_items) / len(true_items)\n    return precision, recall\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:23:35.474992Z","iopub.execute_input":"2025-08-14T14:23:35.475296Z","iopub.status.idle":"2025-08-14T14:23:35.482619Z","shell.execute_reply.started":"2025-08-14T14:23:35.475275Z","shell.execute_reply":"2025-08-14T14:23:35.481526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_users = test_data['customer_id'].unique()[:5000]\n\nprecisions, recalls = [], []\nfor cid in sample_users:\n    p, r = evaluate_precision_recall(cid, k=5, recommend_func=get_cbf_recommendations_fast)\n    if p is not None:\n        precisions.append(p)\n        recalls.append(r)\n\nprint(\"Average Precision@5:\", np.mean(precisions))\nprint(\"Average Recall@5:\", np.mean(recalls))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T14:32:16.892651Z","iopub.execute_input":"2025-08-14T14:32:16.893874Z","iopub.status.idle":"2025-08-14T14:55:19.111526Z","shell.execute_reply.started":"2025-08-14T14:32:16.893832Z","shell.execute_reply":"2025-08-14T14:55:19.110378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\n# === Giả sử đây là kết quả bạn đo được ===\nmetrics = ['Precision@5', 'Recall@5', 'MAP@5']\ncbf_scores = [0.0, 0.0, 0.0]   # CBF result\ncf_scores = [0.27, 0.31, 0.25] # CF result\n\nx = np.arange(len(metrics))  # vị trí cột\nwidth = 0.35                 # độ rộng cột\n\nfig, ax = plt.subplots(figsize=(8, 6))\nbars1 = ax.bar(x - width/2, cbf_scores, width, label='CBF')\nbars2 = ax.bar(x + width/2, cf_scores, width, label='CF')\n\n# Thêm nhãn, tiêu đề\nax.set_ylabel('Score')\nax.set_title('Comparison of CF vs CBF')\nax.set_xticks(x)\nax.set_xticklabels(metrics)\nax.set_ylim(0, 1)\nax.legend()\n\n# Ghi giá trị trên đầu cột\nfor bar in bars1 + bars2:\n    height = bar.get_height()\n    ax.annotate(f'{height:.2f}',\n                xy=(bar.get_x() + bar.get_width() / 2, height),\n                xytext=(0, 3),  # offset lên trên 3 pixel\n                textcoords=\"offset points\",\n                ha='center', va='bottom')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T15:00:05.071042Z","iopub.execute_input":"2025-08-14T15:00:05.071459Z","iopub.status.idle":"2025-08-14T15:00:05.381195Z","shell.execute_reply.started":"2025-08-14T15:00:05.071434Z","shell.execute_reply":"2025-08-14T15:00:05.380133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport random\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom scipy.sparse import csr_matrix\n\n# New: Precompute set of article_ids that have images\nimage_dir = '/kaggle/input/h-and-m-personalized-fashion-recommendations/images'\nexisting_articles = set()\nfor root, dirs, files in os.walk(image_dir):\n    for file in files:\n        if file.endswith('.jpg'):\n            article_str = file[:-4]  # Remove .jpg\n            try:\n                existing_articles.add(int(article_str))\n            except ValueError:\n                pass  # Skip any invalid filenames\n\nprint(f\"Found {len(existing_articles)} articles with images.\")\n\n# ... (continue with your code for user_fav_cluster, item maps, etc.)\n\n# Modified recommendations loop to aim for at least 12 items with images\nrecommendations = {}\nskipped_users = 0\nempty_filtered_items = 0\nuser_gender_map = {}  # Assuming this is defined elsewhere if needed\n\nfor user_idx, user_id in enumerate(user_item_matrix_train.index):\n    fav_cluster = user_fav_cluster.get(user_id, current_season)\n    if fav_cluster is None:\n        skipped_users += 1\n        continue\n    \n    user_gender = user_gender_map.get(user_id, None)\n    preds = predicted_ratings[user_idx, :]\n    item_ids = user_item_matrix_train.columns\n    \n    # Lọc sản phẩm với additional check for existing image\n    item_mask = np.array([\n        item_cluster_map.get(iid, current_season) == fav_cluster and iid in existing_articles\n        for iid in item_ids\n    ])\n    filtered_indices = np.where(item_mask)[0]\n    pred_items_filtered = [\n        (item_ids[i], preds[i] * (1.2 if item_eco_map.get(item_ids[i], False) else 1.0))\n        for i in filtered_indices\n    ]\n    \n    # Sắp xếp theo score descending\n    pred_items_filtered.sort(key=lambda x: x[1], reverse=True)\n    \n    # Nếu ít hơn 12, bổ sung popular items trong cluster, ưu tiên eco, chỉ những có ảnh\n    current_recs = pred_items_filtered[:12]  # Lấy top 12 nếu có nhiều\n    if len(current_recs) < 12:\n        needed = 12 - len(current_recs)\n        already = {iid for iid, _ in current_recs}\n        \n        # Popular eco trong cluster, có ảnh, không trùng\n        popular_eco = train_data[\n            (train_data['season_cluster'] == fav_cluster) &\n            (train_data['eco_label'] == True) &\n            (train_data['article_id'].isin(existing_articles)) &\n            (~train_data['article_id'].isin(already))\n        ]['article_id'].value_counts().index[:needed]\n        current_recs += [(iid, 0.0) for iid in popular_eco]\n        \n        # Nếu vẫn thiếu, popular không eco trong cluster, có ảnh\n        if len(current_recs) < 12:\n            needed = 12 - len(current_recs)\n            already = {iid for iid, _ in current_recs}\n            popular_non_eco = train_data[\n                (train_data['season_cluster'] == fav_cluster) &\n                (train_data['eco_label'] == False) &\n                (train_data['article_id'].isin(existing_articles)) &\n                (~train_data['article_id'].isin(already))\n            ]['article_id'].value_counts().index[:needed]\n            current_recs += [(iid, 0.0) for iid in popular_non_eco]\n        \n        # Nếu vẫn thiếu, popular eco toàn bộ, có ảnh\n        if len(current_recs) < 12:\n            needed = 12 - len(current_recs)\n            already = {iid for iid, _ in current_recs}\n            popular_eco_all = train_data[\n                (train_data['eco_label'] == True) &\n                (train_data['article_id'].isin(existing_articles)) &\n                (~train_data['article_id'].isin(already))\n            ]['article_id'].value_counts().index[:needed]\n            current_recs += [(iid, 0.0) for iid in popular_eco_all]\n        \n        # Nếu vẫn thiếu, popular toàn bộ, có ảnh\n        if len(current_recs) < 12:\n            needed = 12 - len(current_recs)\n            already = {iid for iid, _ in current_recs}\n            popular_all = train_data[\n                (train_data['article_id'].isin(existing_articles)) &\n                (~train_data['article_id'].isin(already))\n            ]['article_id'].value_counts().index[:needed]\n            current_recs += [(iid, 0.0) for iid in popular_all]\n    \n    # Lưu recommendations với ít nhất 12 (hoặc tất cả nếu không đủ)\n    recommendations[user_id] = current_recs[:12]\n\n# ... (rest of your code for evaluation, etc.)\n\n# Hàm lấy đường dẫn hình ảnh (unchanged)\ndef get_image_path(article_id):\n    article_str = str(article_id).zfill(10)\n    folder = article_str[0:3]\n    image_path = Path(f'/kaggle/input/h-and-m-personalized-fashion-recommendations/images/{folder}/{article_str}.jpg')\n    if image_path.exists():\n        return str(image_path)\n    else:\n        return None\n\n# Display code (bây giờ recommendations đã có ít nhất 12, nhưng lọc lại để chắc chắn)\nrandom_user_id = random.choice(list(recommendations.keys()))\nprint(f\"Khách hàng ngẫu nhiên: {random_user_id}\")\n\nfav_cluster = user_fav_cluster.get(random_user_id, 0)\nseason_name = \"Summer\" if fav_cluster == 0 else f\"Season {fav_cluster}\"\n\nrecs = recommendations[random_user_id]\n\n# Lọc lại để chắc chắn tất cả sản phẩm có ảnh (dù đã lọc trước)\nrecs = [(iid, score) for iid, score in recs if get_image_path(iid)]\n\n# Cắt về tối đa 12\nrecs = recs[:12]\nn_items = len(recs)\nnrows = (n_items + 2) // 3\n\nfig, axs = plt.subplots(nrows, 3, figsize=(12, 4*nrows))\nfig.suptitle(f\"{season_name}?\", fontsize=20)\nfor i, (article_id, score) in enumerate(recs):\n    row, col = divmod(i, 3)\n    ax = axs[row, col]\n   \n    image_path = get_image_path(article_id)\n    if image_path:  # Now guaranteed by filter, but keep for safety\n        img = mpimg.imread(image_path)\n        ax.imshow(img)\n    else:\n        ax.text(0.5, 0.5, f\"No Image\\n{article_id}\", ha='center', va='center', fontsize=12)\n        ax.set_facecolor('lightgray')\n    \n    ax.set_title(f\"{article_id}\\nScore: {score:.2f}\", fontsize=10)\n    ax.axis('off')\n    ax.set_xticks([])\n    ax.set_yticks([])\n\n# Ẩn các axes không dùng tới\nfor j in range(i+1, nrows*3):\n    row, col = divmod(j, 3)\n    axs[row, col].axis('off')\n\nplt.tight_layout()\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T18:15:05.176476Z","iopub.execute_input":"2025-08-14T18:15:05.176911Z","iopub.status.idle":"2025-08-14T18:24:20.520701Z","shell.execute_reply.started":"2025-08-14T18:15:05.176885Z","shell.execute_reply":"2025-08-14T18:24:20.519467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"cbf-cách 2-chạy rất lâu không ra kết quả","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport numpy as np\n\n# === 1. Tạo features đơn giản để TF-IDF ===\ntrain_data['features'] = (\n    train_data['product_group_name'].fillna('') + ' ' +\n    train_data['season'].fillna('') + ' ' +\n    train_data['final_gender'].fillna('')\n)\n\n# === 2. TF-IDF ===\nvectorizer = TfidfVectorizer(stop_words='english', max_features=2000)\ntfidf_matrix = vectorizer.fit_transform(train_data['features'])\n\n# === 3. Mapping article_id -> index ===\narticle_id_to_idx = {aid: idx for idx, aid in enumerate(train_data['article_id'])}\n\n# === 4. Cache toàn bộ gợi ý (dùng cosine similarity) ===\ncbf_cache = {}\ndef get_cbf_recommendations_cached(article_id, top_n=20):\n    if article_id in cbf_cache:\n        return cbf_cache[article_id]\n    \n    if article_id not in article_id_to_idx:\n        return []\n    \n    idx = article_id_to_idx[article_id]\n    cosine_sim = cosine_similarity(tfidf_matrix[idx], tfidf_matrix).flatten()\n    similar_indices = np.argsort(-cosine_sim)[1:top_n+1]\n    recs = [(train_data.iloc[i]['article_id'], float(cosine_sim[i])) for i in similar_indices]\n    cbf_cache[article_id] = recs\n    return recs\n\n# === 5. Hàm đánh giá nhiều sản phẩm input của user ===\ndef evaluate_precision_recall_multi(customer_id, k=5):\n    true_items = set(test_data[test_data['customer_id'] == customer_id]['article_id'])\n    if not true_items:\n        return None, None\n    \n    # Lấy nhiều sản phẩm user đã mua trong train\n    start_items = train_data[train_data['customer_id'] == customer_id]['article_id'].tolist()\n    if not start_items:\n        return None, None\n    \n    # Gộp gợi ý từ tất cả start_items\n    rec_scores = {}\n    for item in start_items:\n        for aid, score in get_cbf_recommendations_cached(item, top_n=20):\n            rec_scores[aid] = max(score, rec_scores.get(aid, 0))\n    \n    # Lấy top-k cuối cùng\n    top_recs = sorted(rec_scores.items(), key=lambda x: x[1], reverse=True)[:k]\n    recommended_items = set(aid for aid, score in top_recs)\n    \n    precision = len(recommended_items & true_items) / k\n    recall = len(recommended_items & true_items) / len(true_items)\n    \n    return precision, recall\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T08:16:10.295143Z","iopub.execute_input":"2025-08-14T08:16:10.295510Z","iopub.status.idle":"2025-08-14T08:16:20.086274Z","shell.execute_reply.started":"2025-08-14T08:16:10.295484Z","shell.execute_reply":"2025-08-14T08:16:20.085053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"precisions, recalls = [], []\nfor customer_id in test_data['customer_id'].unique()[:5000]:  # Giữ sample 5000 user\n    p, r = evaluate_precision_recall_multi(customer_id, k=5)\n    if p is not None:\n        precisions.append(p)\n        recalls.append(r)\n\nprint(\"Average Precision@5:\", np.mean(precisions))\nprint(\"Average Recall@5:\", np.mean(recalls))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T08:16:30.951771Z","iopub.execute_input":"2025-08-14T08:16:30.952121Z","iopub.status.idle":"2025-08-14T09:31:32.194339Z","shell.execute_reply.started":"2025-08-14T08:16:30.952096Z","shell.execute_reply":"2025-08-14T09:31:32.192854Z"}},"outputs":[],"execution_count":null}]}