{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-01T12:21:28.057540Z","iopub.execute_input":"2023-01-01T12:21:28.057877Z","iopub.status.idle":"2023-01-01T12:21:28.065413Z","shell.execute_reply.started":"2023-01-01T12:21:28.057850Z","shell.execute_reply":"2023-01-01T12:21:28.064413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pd.read_json('/kaggle/input/otto-recommender-system/train.jsonl',lines=True)\ntest = pd.read_json('/kaggle/input/otto-recommender-system/test.jsonl',lines=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:21:30.308801Z","iopub.execute_input":"2023-01-01T12:21:30.309991Z","iopub.status.idle":"2023-01-01T12:21:51.854559Z","shell.execute_reply.started":"2023-01-01T12:21:30.309931Z","shell.execute_reply":"2023-01-01T12:21:51.853441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 转换格式方便后续处理\ntype_confer={\"clicks\":0,\"carts\":1,\"orders\":2}\nsession_inform = []\nfor i in test.index:\n    session = test.loc[i, \"session\"]\n    events = test.loc[i, \"events\"]\n    for event in events:\n        session_inform.append([session,event[\"aid\"],event[\"ts\"],type_confer[event[\"type\"]]])\ntest_df=pd.DataFrame(session_inform,columns=['session','aid','ts','type'])","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:21:51.856429Z","iopub.execute_input":"2023-01-01T12:21:51.856901Z","iopub.status.idle":"2023-01-01T12:23:04.930302Z","shell.execute_reply.started":"2023-01-01T12:21:51.856864Z","shell.execute_reply":"2023-01-01T12:23:04.928866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"./test_df.npy\",test_df)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:27:21.435954Z","iopub.execute_input":"2023-01-01T12:27:21.436296Z","iopub.status.idle":"2023-01-01T12:27:21.570955Z","shell.execute_reply.started":"2023-01-01T12:27:21.436268Z","shell.execute_reply":"2023-01-01T12:27:21.570068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:27:25.214646Z","iopub.execute_input":"2023-01-01T12:27:25.214993Z","iopub.status.idle":"2023-01-01T12:27:25.224167Z","shell.execute_reply.started":"2023-01-01T12:27:25.214965Z","shell.execute_reply":"2023-01-01T12:27:25.223203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nfrom collections import defaultdict\nimport math\nfrom operator import itemgetter\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:27:54.082024Z","iopub.execute_input":"2023-01-01T12:27:54.082336Z","iopub.status.idle":"2023-01-01T12:27:54.086659Z","shell.execute_reply.started":"2023-01-01T12:27:54.082312Z","shell.execute_reply":"2023-01-01T12:27:54.085622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def itemCFTrain(df):\n    \n    # create list for dict\n    user_item_list = []\n    for idx, row in tqdm(df.iterrows(), total=df.shape[0]):\n        user = int(row['session'])\n        item = int(row['aid'])\n        user_item_list.append([user, item])\n    \n    # create dict\n    user_item_dict = dict()\n    for user, item in tqdm(user_item_list):\n         # change set() to list()\n        user_item_dict.setdefault(user, list())\n        user_item_dict[user].append(item)\n    \n    return user_item_dict","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:27:57.346367Z","iopub.execute_input":"2023-01-01T12:27:57.346777Z","iopub.status.idle":"2023-01-01T12:27:57.353724Z","shell.execute_reply.started":"2023-01-01T12:27:57.346742Z","shell.execute_reply":"2023-01-01T12:27:57.352607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ItemMatrix_fn(user_item_dict):\n    \n    # 采用python字典存储稀疏矩阵\n    # N[i]表示物品i被操作（包括clicks，carts和orders）的次数\n    N = defaultdict(int)\n    itemMatrix = defaultdict(int)\n    for user, items in tqdm(user_item_dict.items()):\n        for i in items:\n            itemMatrix.setdefault(i, dict())\n            N[i] += 1\n            for j in items:\n                itemMatrix[i].setdefault(j, 0)\n                # 若aid i与j同时出现，则将物品相似度矩阵i行j列递增1，否则不存储于稀疏矩阵中\n                itemMatrix[i][j] += 1\n    \n    return itemMatrix, N","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:28:08.150546Z","iopub.execute_input":"2023-01-01T12:28:08.150913Z","iopub.status.idle":"2023-01-01T12:28:08.157595Z","shell.execute_reply.started":"2023-01-01T12:28:08.150883Z","shell.execute_reply":"2023-01-01T12:28:08.156909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ItemSimilarityMatrix_fn(ItemMatrix, N):\n    \n    itemSimMatrix = defaultdict(int)\n    # cosine similarity\n    for i, related_items in tqdm(ItemMatrix.items()):\n        itemSimMatrix.setdefault(i, dict())\n        for j, cij in related_items.items():\n            itemSimMatrix[i].setdefault(j, 0)\n            itemSimMatrix[i][j] = cij / math.sqrt(N[i] * N[j])\n    \n    # normalization\n    for i, relations in tqdm(itemSimMatrix.items()):\n        max_num = relations[max(relations, key=relations.get)]\n        if max_num == 0:\n            continue\n        itemSimMatrix[i] = {k : v / max_num for k, v in relations.items()}\n    \n    return itemSimMatrix","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:28:33.592635Z","iopub.execute_input":"2023-01-01T12:28:33.593015Z","iopub.status.idle":"2023-01-01T12:28:33.600418Z","shell.execute_reply.started":"2023-01-01T12:28:33.592986Z","shell.execute_reply":"2023-01-01T12:28:33.599652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uidict = itemCFTrain(test_df)\nitemMatrix, N = ItemMatrix_fn(uidict)\nitemSimMatrix = ItemSimilarityMatrix_fn(itemMatrix, N)\nnp.save(\"./uidict.npy\",uidict)\nnp.save(\"./itemMatrix.npy\",itemMatrix)\nnp.save(\"./N.npy\",N)\nnp.save(\"./itemSimMatrix.npy\",itemSimMatrix)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:28:37.494453Z","iopub.execute_input":"2023-01-01T12:28:37.494819Z","iopub.status.idle":"2023-01-01T12:36:39.123477Z","shell.execute_reply.started":"2023-01-01T12:28:37.494787Z","shell.execute_reply":"2023-01-01T12:36:39.121148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend(trainData, itemSimMatrix, user, popularity):\n\n    recommends = dict()\n    items = trainData[user]\n    for item in items:\n        # for every item in session, get top 100 similarity scores\n        for i, sim in sorted(itemSimMatrix[item].items(), key=itemgetter(1), reverse=True)[:100]:\n            recommends.setdefault(i, 0.)\n            recommends[i] += sim\n    # sort and return top 20\n    result = list(dict(sorted(recommends.items(), key=itemgetter(1), reverse=True)[:20]).keys())\n    if len(result) < 20:\n        result = result + popularity # if num of items < 20, use most popular items overall\n        result = result[:20]\n    \n    return result","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:36:39.126442Z","iopub.execute_input":"2023-01-01T12:36:39.127629Z","iopub.status.idle":"2023-01-01T12:36:39.134305Z","shell.execute_reply.started":"2023-01-01T12:36:39.127569Z","shell.execute_reply":"2023-01-01T12:36:39.133594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uidict = np.load('./uidict.npy', allow_pickle='TRUE')\nitemMatrix = np.load('./itemMatrix.npy', allow_pickle='TRUE')\nN = np.load('./N.npy', allow_pickle='TRUE')\nitemSimMatrix = np.load('./itemSimMatrix.npy', allow_pickle='TRUE')","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:36:39.135243Z","iopub.execute_input":"2023-01-01T12:36:39.136412Z","iopub.status.idle":"2023-01-01T12:36:57.819574Z","shell.execute_reply.started":"2023-01-01T12:36:39.136385Z","shell.execute_reply":"2023-01-01T12:36:57.817879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uidict=uidict.item()\nitemMatrix=itemMatrix.item()\nN=N.item()\nitemSimMatrix=itemSimMatrix.item()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:36:57.912493Z","iopub.execute_input":"2023-01-01T12:36:57.913086Z","iopub.status.idle":"2023-01-01T12:36:57.917199Z","shell.execute_reply.started":"2023-01-01T12:36:57.913058Z","shell.execute_reply":"2023-01-01T12:36:57.916548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pop is a list of most popular items (in terms of aids) overall\npop = list(dict(sorted(N.items(), key=itemgetter(1), reverse=True)[:20]).keys())\n# users is the IDs of test session\nusers = list(uidict.keys())\nre_items = []\n\nfor user in tqdm(users):\n    result = list(map(str,recommend(uidict, itemSimMatrix, user, pop)))\n    re_items.append(\" \".join(result))\nnp.save(\"./re_items.npy\",re_items)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:36:57.918257Z","iopub.execute_input":"2023-01-01T12:36:57.918629Z","iopub.status.idle":"2023-01-01T12:58:43.239130Z","shell.execute_reply.started":"2023-01-01T12:36:57.918605Z","shell.execute_reply":"2023-01-01T12:58:43.238182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_types = list(np.tile(['_clicks', '_carts', '_orders'], len(users)))\nsub_users = list(map(str, list(np.repeat(users, 3))))\nsub_sessions = [i + j for i, j in zip(sub_users, sub_types)]\nsub_items = list(np.repeat(re_items, 3))\n\nsub = pd.DataFrame({\n    'session_type': sub_sessions,\n    'labels': sub_items\n})","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:27:51.351384Z","iopub.execute_input":"2023-01-01T13:27:51.351797Z","iopub.status.idle":"2023-01-01T13:28:05.709379Z","shell.execute_reply.started":"2023-01-01T13:27:51.351764Z","shell.execute_reply":"2023-01-01T13:28:05.707383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:28:05.712476Z","iopub.execute_input":"2023-01-01T13:28:05.712928Z","iopub.status.idle":"2023-01-01T13:28:05.727827Z","shell.execute_reply.started":"2023-01-01T13:28:05.712885Z","shell.execute_reply":"2023-01-01T13:28:05.726544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:28:05.729352Z","iopub.execute_input":"2023-01-01T13:28:05.730216Z","iopub.status.idle":"2023-01-01T13:28:20.157427Z","shell.execute_reply.started":"2023-01-01T13:28:05.730178Z","shell.execute_reply":"2023-01-01T13:28:20.156068Z"},"trusted":true},"execution_count":null,"outputs":[]}]}