{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"\nH&M个性化时尚推荐竞赛 - 精简高效解决方案\n主要步骤：\n1. 数据加载和预处理\n2. 特征工程和数据准备\n3. 协同过滤模型训练\n4. 预测生成和提交文件创建\n\"\"\"\n\n# 安装必要的库\nprint(\"安装必要的库...\")\n!pip install implicit -q\n\n# 导入所需库\nprint(\"导入必要的库...\")\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom datetime import datetime, timedelta\nfrom scipy.sparse import csr_matrix\nimport gc\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# 尝试导入implicit库，如果失败则使用替代方案\ntry:\n    from implicit.als import AlternatingLeastSquares\n    print(\"成功导入implicit库\")\n    use_implicit = True\nexcept:\n    from sklearn.decomposition import NMF\n    print(\"使用NMF作为替代方案\")\n    use_implicit = False\n\n# 设置随机种子，确保结果可复现\nnp.random.seed(42)\n\n# 记录开始时间\nstart_time = datetime.now()\nprint(f\"开始执行 - 当前时间: {start_time.strftime('%Y-%m-%d %H:%M:%S')}\")\n\n# 设置数据路径\nBASE_PATH = '/kaggle/input/h-and-m-personalized-fashion-recommendations'\nTRANSACTIONS_PATH = f'{BASE_PATH}/transactions_train.csv'\nARTICLES_PATH = f'{BASE_PATH}/articles.csv'\nCUSTOMERS_PATH = f'{BASE_PATH}/customers.csv'\nSAMPLE_SUB_PATH = f'{BASE_PATH}/sample_submission.csv'\n\n# -----------------------------------------------------------------------------\n# 第1部分：数据加载与基础预处理\n# -----------------------------------------------------------------------------\n\nprint(\"\\n开始数据加载与预处理...\")\n\n# 加载数据\ntransactions = pd.read_csv(TRANSACTIONS_PATH)\narticles = pd.read_csv(ARTICLES_PATH)\ncustomers = pd.read_csv(CUSTOMERS_PATH)\nsample_submission = pd.read_csv(SAMPLE_SUB_PATH)\n\n# 显示数据基本信息\nprint(f\"交易数据大小: {transactions.shape}\")\nprint(f\"商品数据大小: {articles.shape}\")\nprint(f\"客户数据大小: {customers.shape}\")\nprint(f\"需要预测的用户数: {len(sample_submission['customer_id'].unique())}\")\n\n# 时间处理\ntransactions['t_dat'] = pd.to_datetime(transactions['t_dat'])\nlast_date = transactions['t_dat'].max()\nprint(f\"数据集最后日期: {last_date}\")\n\n# 确保商品ID格式正确（整数格式，稍后再转换为字符串）\ndef ensure_int(x):\n    return int(x)\n\ntransactions['article_id'] = transactions['article_id'].apply(ensure_int)\narticles['article_id'] = articles['article_id'].apply(ensure_int)\n\n# -----------------------------------------------------------------------------\n# 第2部分：特征工程和数据预处理\n# -----------------------------------------------------------------------------\n\nprint(\"\\n开始特征工程...\")\n\n# 计算时间窗口\nlast_week_date = last_date - timedelta(days=7)\nlast_2weeks_date = last_date - timedelta(days=14)\ncutoff_date = last_date - timedelta(days=90)  # 只使用最近90天的数据\n\n# 筛选最近的交易数据\nrecent_transactions = transactions[transactions['t_dat'] >= cutoff_date].copy()\nprint(f\"最近90天交易记录数: {len(recent_transactions)}\")\n\n# 计算不同时间窗口的热门商品\nprint(\"\\n计算热门商品...\")\n\n# 最后一周的热门商品\nlast_week_data = recent_transactions[recent_transactions['t_dat'] >= last_week_date]\nlast_week_popular = last_week_data['article_id'].value_counts().head(20).index.tolist()\n\n# 最后两周的热门商品\nlast_2weeks_data = recent_transactions[recent_transactions['t_dat'] >= last_2weeks_date]\nlast_2weeks_popular = last_2weeks_data['article_id'].value_counts().head(20).index.tolist()\n\n# 全局热门商品\npopular_items = recent_transactions['article_id'].value_counts().head(30).index.tolist()\n\nprint(f\"最后一周热门商品数: {len(last_week_popular)}\")\nprint(f\"最后两周热门商品数: {len(last_2weeks_popular)}\")\nprint(f\"全局热门商品数: {len(popular_items)}\")\n\n# 计算用户购买历史\nprint(\"\\n计算用户购买历史...\")\n\n# 用户最近购买的商品\nuser_last_purchases = {}\nfor customer, group in tqdm(recent_transactions.groupby('customer_id')):\n    # 按时间排序，获取最近购买的商品\n    user_items = group.sort_values('t_dat', ascending=False)['article_id'].tolist()\n    user_last_purchases[customer] = user_items[:12]\n\n# 用户最后一周购买的商品\nuser_last_week_purchases = {}\nfor customer, group in tqdm(last_week_data.groupby('customer_id')):\n    user_items = group['article_id'].unique().tolist()\n    user_last_week_purchases[customer] = user_items\n\nprint(f\"有购买历史的用户数: {len(user_last_purchases)}\")\nprint(f\"最后一周有购买的用户数: {len(user_last_week_purchases)}\")\n\n# 释放内存\ndel last_week_data, last_2weeks_data\ngc.collect()\n\n# -----------------------------------------------------------------------------\n# 第3部分：协同过滤模型训练\n# -----------------------------------------------------------------------------\n\nprint(\"\\n准备协同过滤模型...\")\n\n# 创建用户-物品交互矩阵\nui_data = recent_transactions.groupby(['customer_id', 'article_id']).size().reset_index(name='count')\n\n# 为了减少内存使用，使用字典映射用户ID和物品ID\ncustomer_ids = ui_data['customer_id'].unique()\narticle_ids = ui_data['article_id'].unique()\nprint(f\"协同过滤模型中的用户数: {len(customer_ids)}\")\nprint(f\"协同过滤模型中的商品数: {len(article_ids)}\")\n\n# 创建ID映射字典\ncustomer_map = {cid: i for i, cid in enumerate(customer_ids)}\narticle_map = {aid: i for i, aid in enumerate(article_ids)}\nreverse_customer_map = {i: cid for cid, i in customer_map.items()}\nreverse_article_map = {i: aid for aid, i in article_map.items()}\n\n# 应用映射\nui_data['cid'] = ui_data['customer_id'].map(customer_map)\nui_data['aid'] = ui_data['article_id'].map(article_map)\n\n# 创建稀疏矩阵\nprint(\"创建用户-商品交互稀疏矩阵...\")\nsparse_ui_matrix = csr_matrix((ui_data['count'], (ui_data['cid'], ui_data['aid'])))\nprint(f\"稀疏矩阵形状: {sparse_ui_matrix.shape}\")\n\n# 初始化模型推荐结果\ncf_recs = {}\n\n# 根据库的可用性选择不同的协同过滤方法\nif use_implicit:\n    # 使用implicit库的ALS模型\n    print(\"\\n训练ALS协同过滤模型...\")\n    model = AlternatingLeastSquares(\n        factors=200,            # 增加因子数量\n        regularization=0.01,    # 减小正则化参数\n        iterations=30,          # 增加迭代次数 \n        calculate_training_loss=True,\n        random_state=42\n    )\n    model.fit(sparse_ui_matrix, show_progress=True)\n    \n    # 确定需要预测的用户集合\n    print(\"\\n预计算用户推荐...\")\n    test_customers = set(sample_submission['customer_id'].unique())\n    # 找出在训练集中有数据的测试用户\n    users_in_train = set(customer_ids).intersection(test_customers)\n    print(f\"训练集中存在的测试用户数: {len(users_in_train)}\")\n    \n    # 为训练集中的用户预计算推荐\n    for customer_id in tqdm(users_in_train):\n        try:\n            # 获取映射后的用户ID\n            cid = customer_map[customer_id]\n            # 获取用户的推荐结果\n            recs = model.recommend(cid, sparse_ui_matrix[cid], N=20)\n            # 转换回原始商品ID\n            rec_items = [reverse_article_map[item_id] for item_id, _ in recs]\n            cf_recs[customer_id] = rec_items\n        except:\n            continue\nelse:\n    # 使用scikit-learn的NMF作为替代\n    print(\"\\n训练NMF协同过滤模型（替代方案）...\")\n    model = NMF(\n        n_components=100,       # 增加组件数\n        init='random',\n        random_state=42,\n        max_iter=50,            # 增加迭代次数\n        alpha=0.01\n    )\n    \n    # 选择最活跃的用户子集进行训练\n    max_users = min(100000, sparse_ui_matrix.shape[0])\n    user_activity = np.asarray(sparse_ui_matrix.sum(axis=1)).flatten()\n    active_users = np.argsort(-user_activity)[:max_users]\n    \n    # 只训练活跃用户子集\n    print(f\"使用 {max_users} 个最活跃用户训练NMF模型...\")\n    active_matrix = sparse_ui_matrix[active_users, :].toarray()\n    model.fit(active_matrix)\n    \n    # 计算商品特征\n    item_features = model.components_.T\n    \n    # 为活跃用户生成推荐\n    print(\"\\n为活跃用户生成NMF推荐...\")\n    for i, user_idx in enumerate(tqdm(active_users)):\n        # 获取原始用户ID\n        customer_id = reverse_customer_map[user_idx]\n        \n        # 计算用户-商品得分\n        if active_matrix[i].sum() > 0:  # 确保用户有交互\n            user_vector = active_matrix[i]\n            user_features = model.transform([user_vector])[0]\n            scores = np.dot(user_features, item_features.T)\n            \n            # 过滤掉已交互商品\n            interacted_items = set(np.where(user_vector > 0)[0])\n            scores_with_idx = [(scores[j], j) for j in range(len(scores)) if j not in interacted_items]\n            \n            # 获取得分最高的商品\n            top_items_idx = [idx for _, idx in sorted(scores_with_idx, reverse=True)[:20]]\n            # 转换回原始商品ID\n            rec_items = [reverse_article_map[item_idx] for item_idx in top_items_idx]\n            cf_recs[customer_id] = rec_items\n\nprint(f\"成功为 {len(cf_recs)} 个用户预计算了推荐\")\n\n# 释放内存\ndel recent_transactions, ui_data, sparse_ui_matrix\nif 'active_matrix' in locals():\n    del active_matrix\ngc.collect()\n\n# -----------------------------------------------------------------------------\n# 第4部分：生成最终推荐并创建提交文件\n# -----------------------------------------------------------------------------\n\nprint(\"\\n生成最终推荐...\")\n\n# 确保预测格式正确的验证函数\ndef validate_prediction_format(pred_str):\n    \"\"\"验证预测格式是否正确\"\"\"\n    items = pred_str.split()\n    if len(items) != 12:\n        return False\n    return True\n\n# 单个用户推荐生成函数\ndef generate_customer_prediction(customer_id):\n    \"\"\"为单个客户生成预测\"\"\"\n    candidate_items = []\n    \n    # 1. 优先考虑用户最近一周购买的商品\n    last_week_items = user_last_week_purchases.get(customer_id, [])\n    for item in last_week_items:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 2. 然后添加用户历史购买记录中的商品\n    user_items = user_last_purchases.get(customer_id, [])\n    for item in user_items:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 3. 添加协同过滤推荐的商品\n    cf_items = cf_recs.get(customer_id, [])\n    for item in cf_items:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 4. 添加最后一周的热门商品\n    for item in last_week_popular:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 5. 添加最后两周的热门商品\n    for item in last_2weeks_popular:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 6. 最后添加全局热门商品\n    for item in popular_items:\n        if item not in candidate_items and len(candidate_items) < 12:\n            candidate_items.append(item)\n    \n    # 确保有12个推荐\n    final_items = candidate_items[:12]\n    \n    # 如果还不够12个，用热门商品填充\n    while len(final_items) < 12:\n        for item in popular_items:\n            if item not in final_items:\n                final_items.append(item)\n                if len(final_items) >= 12:\n                    break\n    \n    # 确保商品ID格式正确 - 补充前导零至10位\n    formatted_items = [f\"{int(item):010d}\" for item in final_items[:12]]\n    prediction = ' '.join(formatted_items)\n    \n    return prediction\n\n# 为所有测试用户生成推荐\ntest_customers = sample_submission['customer_id'].unique()\nresults = []\n\nfor customer in tqdm(test_customers):\n    prediction = generate_customer_prediction(customer)\n    results.append((customer, prediction))\n\n# 创建提交DataFrame\nsubmission = pd.DataFrame(results, columns=['customer_id', 'prediction'])\n\n# 验证提交文件格式\nprint(\"\\n验证提交文件格式...\")\ninvalid_predictions = submission[~submission['prediction'].apply(validate_prediction_format)]\nif len(invalid_predictions) > 0:\n    print(f\"警告：发现 {len(invalid_predictions)} 个格式不正确的预测\")\n    print(\"示例：\")\n    print(invalid_predictions.head())\nelse:\n    print(\"所有预测格式正确\")\n\n# 保存提交文件\nsubmission.to_csv('submission.csv', index=False)\n\n# 最终验证\nprint(\"\\n最终验证...\")\nfinal_submission = pd.read_csv('submission.csv')\nprint(f\"提交文件行数: {len(final_submission)}\")\nprint(\"预测示例:\")\nprint(final_submission['prediction'].head())\n\n# 记录结束时间\nend_time = datetime.now()\nprint(f\"\\n完成! 已生成提交文件 'submission.csv'\")\nprint(f\"总运行时间: {(end_time - start_time).total_seconds() / 60:.2f} 分钟\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-28T09:54:28.013426Z","iopub.execute_input":"2025-03-28T09:54:28.013877Z","execution_failed":"2025-03-28T09:56:48.168Z"}},"outputs":[],"execution_count":null}]}