{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"},{"sourceId":4436180,"sourceType":"datasetVersion","datasetId":2597726}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 实验2 （1）补全代码：不分行为使用itemCF ","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:53:02.080517Z","iopub.execute_input":"2024-06-09T03:53:02.081210Z","iopub.status.idle":"2024-06-09T03:53:02.087656Z","shell.execute_reply.started":"2024-06-09T03:53:02.081174Z","shell.execute_reply":"2024-06-09T03:53:02.086469Z"}}},{"cell_type":"code","source":"import pandas as pd, numpy as np\n\nimport glob\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom datetime import datetime\n\nfrom tqdm import tqdm\n\ntype_labels = {'clicks':1, 'carts':2, 'orders':3}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def load(which):    \n    dfs = []\n\n    test_files = glob.glob('/kaggle/input/otto-chunk-data-inparquet-format/'+which+'_parquet/*')\n    \n    for e, chunk_file in enumerate(test_files):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n        \n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:53:02.091587Z","iopub.execute_input":"2024-06-09T03:53:02.092055Z","iopub.status.idle":"2024-06-09T03:53:02.100876Z","shell.execute_reply.started":"2024-06-09T03:53:02.092013Z","shell.execute_reply":"2024-06-09T03:53:02.099359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load('train')","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:53:02.102957Z","iopub.execute_input":"2024-06-09T03:53:02.103386Z","iopub.status.idle":"2024-06-09T03:54:12.091143Z","shell.execute_reply.started":"2024-06-09T03:53:02.103344Z","shell.execute_reply":"2024-06-09T03:54:12.089874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To reduce runtime, only consider data from the last four days as train_df.","metadata":{}},{"cell_type":"code","source":"datetime.strptime(\"2022-08-25 00:00:00\", \"%Y-%m-%d %H:%M:%S\").timestamp()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:54:12.093611Z","iopub.execute_input":"2024-06-09T03:54:12.094072Z","iopub.status.idle":"2024-06-09T03:54:12.104535Z","shell.execute_reply.started":"2024-06-09T03:54:12.094031Z","shell.execute_reply":"2024-06-09T03:54:12.103168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df[train_df['ts']>=1661385600].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:54:12.106070Z","iopub.execute_input":"2024-06-09T03:54:12.106413Z","iopub.status.idle":"2024-06-09T03:54:14.155795Z","shell.execute_reply.started":"2024-06-09T03:54:12.106382Z","shell.execute_reply":"2024-06-09T03:54:14.153757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['datetime'] = train_df['ts'].apply(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:54:14.162442Z","iopub.execute_input":"2024-06-09T03:54:14.163710Z","iopub.status.idle":"2024-06-09T03:55:31.181741Z","shell.execute_reply.started":"2024-06-09T03:54:14.163636Z","shell.execute_reply":"2024-06-09T03:55:31.180357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choose the last day as the validation set.","metadata":{}},{"cell_type":"code","source":"train_df['datetime'].dt.day.unique()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:31.183407Z","iopub.execute_input":"2024-06-09T03:55:31.183864Z","iopub.status.idle":"2024-06-09T03:55:32.173556Z","shell.execute_reply.started":"2024-06-09T03:55:31.183797Z","shell.execute_reply":"2024-06-09T03:55:32.172276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['day'] = train_df['datetime'].dt.day","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:32.175304Z","iopub.execute_input":"2024-06-09T03:55:32.175684Z","iopub.status.idle":"2024-06-09T03:55:33.082461Z","shell.execute_reply.started":"2024-06-09T03:55:32.175651Z","shell.execute_reply":"2024-06-09T03:55:33.081396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val_offline = train_df[train_df['day']==28].reset_index(drop=True)\ndf_train_offline = train_df[train_df['day']<28].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:33.083981Z","iopub.execute_input":"2024-06-09T03:55:33.084401Z","iopub.status.idle":"2024-06-09T03:55:36.034514Z","shell.execute_reply.started":"2024-06-09T03:55:33.084348Z","shell.execute_reply":"2024-06-09T03:55:36.033252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A very simple itemCF for now; later, improvements and optimizations will be considered, such as time factors, click order, and user activity.","metadata":{}},{"cell_type":"code","source":"df_train_offline","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:36.035983Z","iopub.execute_input":"2024-06-09T03:55:36.036313Z","iopub.status.idle":"2024-06-09T03:55:36.059687Z","shell.execute_reply.started":"2024-06-09T03:55:36.036285Z","shell.execute_reply":"2024-06-09T03:55:36.058358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_offline['aid'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:36.061413Z","iopub.execute_input":"2024-06-09T03:55:36.062092Z","iopub.status.idle":"2024-06-09T03:55:36.549426Z","shell.execute_reply.started":"2024-06-09T03:55:36.062036Z","shell.execute_reply":"2024-06-09T03:55:36.548216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_offline","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:36.555044Z","iopub.execute_input":"2024-06-09T03:55:36.555421Z","iopub.status.idle":"2024-06-09T03:55:36.571180Z","shell.execute_reply.started":"2024-06-09T03:55:36.555390Z","shell.execute_reply":"2024-06-09T03:55:36.570006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df = df_train_offline[['session','aid']].groupby('session',as_index=False).agg(list)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:55:36.572847Z","iopub.execute_input":"2024-06-09T03:55:36.573932Z","iopub.status.idle":"2024-06-09T03:57:07.061825Z","shell.execute_reply.started":"2024-06-09T03:55:36.573891Z","shell.execute_reply":"2024-06-09T03:57:07.060585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df_val = df_val_offline[['session','aid']].groupby('session',as_index=False).agg(list)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:07.063177Z","iopub.execute_input":"2024-06-09T03:57:07.063515Z","iopub.status.idle":"2024-06-09T03:57:50.326852Z","shell.execute_reply.started":"2024-06-09T03:57:07.063485Z","shell.execute_reply":"2024-06-09T03:57:50.325800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df_val","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.328363Z","iopub.execute_input":"2024-06-09T03:57:50.328726Z","iopub.status.idle":"2024-06-09T03:57:50.349056Z","shell.execute_reply.started":"2024-06-09T03:57:50.328690Z","shell.execute_reply":"2024-06-09T03:57:50.347757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.351105Z","iopub.execute_input":"2024-06-09T03:57:50.351441Z","iopub.status.idle":"2024-06-09T03:57:50.375066Z","shell.execute_reply.started":"2024-06-09T03:57:50.351412Z","shell.execute_reply":"2024-06-09T03:57:50.373710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 保留前1000行并重置索引\nsession_aids_df = session_aids_df.head(1000).reset_index(drop=True)\nsession_aids_df","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.376903Z","iopub.execute_input":"2024-06-09T03:57:50.377551Z","iopub.status.idle":"2024-06-09T03:57:50.397678Z","shell.execute_reply.started":"2024-06-09T03:57:50.377507Z","shell.execute_reply":"2024-06-09T03:57:50.396538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.399161Z","iopub.execute_input":"2024-06-09T03:57:50.400217Z","iopub.status.idle":"2024-06-09T03:57:50.406250Z","shell.execute_reply.started":"2024-06-09T03:57:50.400183Z","shell.execute_reply":"2024-06-09T03:57:50.405133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_Sim = defaultdict()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.407873Z","iopub.execute_input":"2024-06-09T03:57:50.408869Z","iopub.status.idle":"2024-06-09T03:57:50.418252Z","shell.execute_reply.started":"2024-06-09T03:57:50.408800Z","shell.execute_reply":"2024-06-09T03:57:50.417075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_Sim","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.419963Z","iopub.execute_input":"2024-06-09T03:57:50.420323Z","iopub.status.idle":"2024-06-09T03:57:50.438157Z","shell.execute_reply.started":"2024-06-09T03:57:50.420291Z","shell.execute_reply":"2024-06-09T03:57:50.436883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,row in tqdm( session_aids_df.iterrows(), total=len(session_aids_df) ):\n    \n    aid_list = row['aid']\n   \n    \n    for item in aid_list:\n        \n        \n        item_Sim.setdefault(item,dict())  # 通过设置default，使得对于item这一个aid，在该字典中存在一个空字典作为其默认值，不会出现 KeyError的错误，即这行代码使用defaultdict的setdefault方法来确保item_Sim字典中存在当前项目item作为键的条目。如果不存在，它将创建一个空字典作为该键的值。\n        \n        \n    \n        for related_item in aid_list:    # 找到相关联的物品     只要他们出现在同一aid列表里，也就是上文提到的在session_aids_df里面aid列中的某个具体的列表里面，就证明他们是有关联的\n            if item == related_item:\n                # 不考虑同一aid物品   不需要计算项目与自身的相似度。\n                continue\n        \n            item_Sim[item].setdefault(related_item,0)      # 确保item_Sim字典中item键对应的字典存在related_item作为键的条目。如果不存在，它将创建一个条目并设置值为0。\n            \n            item_Sim[item][related_item] += 1      # 这行代码将related_item在item的相似度计数加1。这意味着如果多个用户在同一个会话中同时交互了item和related_item，那么这两个项目的相似度会增加。   \n    ","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.439793Z","iopub.execute_input":"2024-06-09T03:57:50.440385Z","iopub.status.idle":"2024-06-09T03:57:50.967776Z","shell.execute_reply.started":"2024-06-09T03:57:50.440328Z","shell.execute_reply":"2024-06-09T03:57:50.966751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import sqrt","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.969196Z","iopub.execute_input":"2024-06-09T03:57:50.969548Z","iopub.status.idle":"2024-06-09T03:57:50.974295Z","shell.execute_reply.started":"2024-06-09T03:57:50.969517Z","shell.execute_reply":"2024-06-09T03:57:50.973020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# item_Sim     # 这一个是字典类型的，输出不了，太多了","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.976152Z","iopub.execute_input":"2024-06-09T03:57:50.976603Z","iopub.status.idle":"2024-06-09T03:57:50.985619Z","shell.execute_reply.started":"2024-06-09T03:57:50.976558Z","shell.execute_reply":"2024-06-09T03:57:50.984451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# item_Sim.items()","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.987275Z","iopub.execute_input":"2024-06-09T03:57:50.987730Z","iopub.status.idle":"2024-06-09T03:57:50.996862Z","shell.execute_reply.started":"2024-06-09T03:57:50.987670Z","shell.execute_reply":"2024-06-09T03:57:50.995525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 计算项目间的相似度\ndef calculate_similarity(item_Sim):\n    similarity_matrix = defaultdict(dict)\n    \n    for item, related_items in tqdm(item_Sim.items(), total=len(item_Sim)):\n        for related_item, count in related_items.items():\n            # 计算与item共同出现的不同项目的总数\n            total_related_items = len(set(related_items))     # total_related_items 是与items相关的aid的总数量\n            # 避免除以零，如果total_related_items为0，则跳过\n            if total_related_items > 0:\n                # 使用Jaccard相似度计算相似度\n                similarity = count / total_related_items\n                similarity_matrix[item][related_item] = similarity\n    \n    return similarity_matrix\n\n# 计算相似度矩阵\nsimilarity_matrix = calculate_similarity(item_Sim)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:50.998094Z","iopub.execute_input":"2024-06-09T03:57:50.998484Z","iopub.status.idle":"2024-06-09T03:57:51.521576Z","shell.execute_reply.started":"2024-06-09T03:57:50.998453Z","shell.execute_reply":"2024-06-09T03:57:51.520487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"similarity_matrix","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:12:08.941077Z","iopub.execute_input":"2024-06-09T04:12:08.941545Z","iopub.status.idle":"2024-06-09T04:12:09.875029Z","shell.execute_reply.started":"2024-06-09T04:12:08.941509Z","shell.execute_reply":"2024-06-09T04:12:09.873744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 我们要定义一个用户历史点击记录，即用户对什么感兴趣，我想他是在训练集上定义，即用户在训练集上没点开过，将没点开过的物品添加到\n# 推荐商品列表里，并根据相似性选择前5个，然后到了验证集上，查看这推荐商品是否出现在了28号的数据集上。\n\n# 我感觉也不用定义，aid_list就存放这用户不同会话的点击的aid列表\n# 我感觉我们要做的就是找到每个会话session对应的aid列表，然后遍历相似矩阵，找到没在aid列表的related_item，将对应的\n","metadata":{"execution":{"iopub.status.busy":"2024-06-09T03:57:51.523299Z","iopub.execute_input":"2024-06-09T03:57:51.524256Z","iopub.status.idle":"2024-06-09T03:57:51.529567Z","shell.execute_reply.started":"2024-06-09T03:57:51.524213Z","shell.execute_reply":"2024-06-09T03:57:51.528189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_recommendations(similarity_matrix, session_aids_df, num_recommendations=20):\n    recommendations = defaultdict(int)\n    all_recommendations = defaultdict(list)    # 存储每个会话的推荐结果\n    for i,row in tqdm( session_aids_df.iterrows(), total=len(session_aids_df) ):  # 我感觉不太需要这个，应当是遍历训练集，将session作为迭代器，遍历每一个session\n    # 针对每一个session，根据相似矩阵选择相关物品中的前20个，作为推荐，然后与真实的28号的验证集作比较，看看我们选中了几个物品\n    # session_aids_df中有 session列 和aid列\n        aid_list = row['aid']\n        user_history = set(aid_list)  # 使用集合避免重复推荐相同的项目\n        for item in user_history:\n            if item in similarity_matrix:\n                for related_item, similarity in similarity_matrix[item].items():\n                    if related_item not in user_history:\n                        recommendations[related_item] += similarity       # 将相似度+=\n             # 根据相似度分数排序推荐项目\n            sorted_recommendations = sorted(recommendations.items(), key=lambda x: x[1], reverse=True)\n            # 获取分数最高的前 num_recommendations 个推荐\n            top_recommendations = [item for item, _ in sorted_recommendations[:num_recommendations]]\n        \n        # 将推荐结果添加到 all_recommendations 中，以会话ID为键\n        all_recommendations[row['session']] = top_recommendations\n    \n    return all_recommendations\n\n# 假设 user_history 是用户过去喜欢过的项目列表\n# user_history = ['item1', 'item2']\n# 生成推荐\nrecommended_items = generate_recommendations(similarity_matrix, session_aids_df)\nprint(\"Recommended items:\", recommended_items)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:06:09.531204Z","iopub.execute_input":"2024-06-09T04:06:09.531618Z","iopub.status.idle":"2024-06-09T04:06:27.352062Z","shell.execute_reply.started":"2024-06-09T04:06:09.531587Z","shell.execute_reply":"2024-06-09T04:06:27.350916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"获得推荐系统之后我们便可以与验证集作比较了","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_items_val = df_val_offline.groupby(['session','type'],as_index=False)['aid'].agg(set)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:06:48.620632Z","iopub.execute_input":"2024-06-09T04:06:48.621055Z","iopub.status.idle":"2024-06-09T04:07:46.253765Z","shell.execute_reply.started":"2024-06-09T04:06:48.621023Z","shell.execute_reply":"2024-06-09T04:07:46.252361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getScore(df, type_, suffix):\n    \n    df_tmp = df[df['type']==type_].reset_index(drop=True)     # df_tmp是真实验证集数据\n    print(df_tmp)\n    score_Numerator = 0\n    score_Denominator = 0\n\n    for i,row in tqdm( df_tmp.iterrows(),total=len(df_tmp) ):\n        # 接下来是我们推荐物品的获取\n        recom_tmp = recommended_items[row['session']]\n    \n        score_Numerator += len(row['aid'] & set(recom_tmp))\n        score_Denominator += min(20,len(row['aid']))\n        \n    return score_Numerator/score_Denominator","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:07:46.256228Z","iopub.execute_input":"2024-06-09T04:07:46.256738Z","iopub.status.idle":"2024-06-09T04:07:46.265711Z","shell.execute_reply.started":"2024-06-09T04:07:46.256683Z","shell.execute_reply":"2024-06-09T04:07:46.264382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 计算得分：\nscore_click = getScore(session_items_val,1,'_clicks')\nscore_cart = getScore(session_items_val,2,'_carts')\nscore_order = getScore(session_items_val,3,'_orders')\nscore_click * 0.1 + score_cart * 0.3 + score_order * 0.6\nprint(score_click)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:07:46.267343Z","iopub.execute_input":"2024-06-09T04:07:46.267885Z","iopub.status.idle":"2024-06-09T04:10:05.011687Z","shell.execute_reply.started":"2024-06-09T04:07:46.267842Z","shell.execute_reply":"2024-06-09T04:10:05.010278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_click * 0.1 + score_cart * 0.3 + score_order * 0.6\nprint(score_click)","metadata":{"execution":{"iopub.status.busy":"2024-06-09T04:14:56.751262Z","iopub.execute_input":"2024-06-09T04:14:56.751793Z","iopub.status.idle":"2024-06-09T04:14:56.758690Z","shell.execute_reply.started":"2024-06-09T04:14:56.751752Z","shell.execute_reply":"2024-06-09T04:14:56.757387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}