{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"},{"sourceId":4436180,"sourceType":"datasetVersion","datasetId":2597726}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I am teaching a hands-on data mining course for undergraduate students at our school, and since Kaggle offers free computing resources, I'm writing some basic code to serve as examples. \n# If you find any mistakes, please leave a comment in the comment section.","metadata":{}},{"cell_type":"markdown","source":"##  [Lession 01 Recommend using popular items](https://www.kaggle.com/code/bruceqdu/lession-01-recommend-using-popular-items/notebook)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\n\nimport glob\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom datetime import datetime\n\nfrom tqdm import tqdm\n\ntype_labels = {'clicks':1, 'carts':2, 'orders':3}","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:47:48.718394Z","iopub.execute_input":"2024-03-16T01:47:48.718736Z","iopub.status.idle":"2024-03-16T01:47:49.082570Z","shell.execute_reply.started":"2024-03-16T01:47:48.718711Z","shell.execute_reply":"2024-03-16T01:47:49.081285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(which):    \n    dfs = []\n\n    test_files = glob.glob('/kaggle/input/otto-chunk-data-inparquet-format/'+which+'_parquet/*')\n    \n    for e, chunk_file in enumerate(test_files):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n        \n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:47:49.084556Z","iopub.execute_input":"2024-03-16T01:47:49.085070Z","iopub.status.idle":"2024-03-16T01:47:49.092222Z","shell.execute_reply.started":"2024-03-16T01:47:49.085036Z","shell.execute_reply":"2024-03-16T01:47:49.090592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load('train')","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:47:49.093580Z","iopub.execute_input":"2024-03-16T01:47:49.093910Z","iopub.status.idle":"2024-03-16T01:48:39.345880Z","shell.execute_reply.started":"2024-03-16T01:47:49.093880Z","shell.execute_reply":"2024-03-16T01:48:39.344325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To reduce runtime, only consider data from the last three days as train_df.","metadata":{}},{"cell_type":"code","source":"datetime.strptime(\"2022-08-25 00:00:00\", \"%Y-%m-%d %H:%M:%S\").timestamp()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:48:39.348331Z","iopub.execute_input":"2024-03-16T01:48:39.348671Z","iopub.status.idle":"2024-03-16T01:48:39.357374Z","shell.execute_reply.started":"2024-03-16T01:48:39.348643Z","shell.execute_reply":"2024-03-16T01:48:39.356088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df[train_df['ts']>=1661385600].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:48:39.358399Z","iopub.execute_input":"2024-03-16T01:48:39.358703Z","iopub.status.idle":"2024-03-16T01:48:40.641079Z","shell.execute_reply.started":"2024-03-16T01:48:39.358675Z","shell.execute_reply":"2024-03-16T01:48:40.639091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['datetime'] = train_df['ts'].apply(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:48:40.642214Z","iopub.execute_input":"2024-03-16T01:48:40.643871Z","iopub.status.idle":"2024-03-16T01:49:16.303081Z","shell.execute_reply.started":"2024-03-16T01:48:40.643837Z","shell.execute_reply":"2024-03-16T01:49:16.301243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choose the last day as the validation set.","metadata":{}},{"cell_type":"code","source":"train_df['datetime'].dt.day.unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:49:16.304361Z","iopub.execute_input":"2024-03-16T01:49:16.304701Z","iopub.status.idle":"2024-03-16T01:49:16.964132Z","shell.execute_reply.started":"2024-03-16T01:49:16.304665Z","shell.execute_reply":"2024-03-16T01:49:16.963199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['day'] = train_df['datetime'].dt.day","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:49:16.967691Z","iopub.execute_input":"2024-03-16T01:49:16.968110Z","iopub.status.idle":"2024-03-16T01:49:17.537225Z","shell.execute_reply.started":"2024-03-16T01:49:16.968073Z","shell.execute_reply":"2024-03-16T01:49:17.535769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val_offline = train_df[train_df['day']==28].reset_index(drop=True)\ndf_train_offline = train_df[train_df['day']<28].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:49:17.539140Z","iopub.execute_input":"2024-03-16T01:49:17.539497Z","iopub.status.idle":"2024-03-16T01:49:19.250361Z","shell.execute_reply.started":"2024-03-16T01:49:17.539470Z","shell.execute_reply":"2024-03-16T01:49:19.249536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A very simple itemCF for now; later, improvements and optimizations will be considered, such as time factors, click order, and user activity.","metadata":{}},{"cell_type":"code","source":"df_train_offline","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:49:52.634000Z","iopub.execute_input":"2024-03-16T01:49:52.634346Z","iopub.status.idle":"2024-03-16T01:49:52.652637Z","shell.execute_reply.started":"2024-03-16T01:49:52.634318Z","shell.execute_reply":"2024-03-16T01:49:52.651360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_offline['aid'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:54:23.992995Z","iopub.execute_input":"2024-03-16T01:54:23.993355Z","iopub.status.idle":"2024-03-16T01:54:24.506644Z","shell.execute_reply.started":"2024-03-16T01:54:23.993327Z","shell.execute_reply":"2024-03-16T01:54:24.505607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df = df_train_offline[['session','aid']].groupby('session',as_index=False).agg(list)","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:52:57.948408Z","iopub.execute_input":"2024-03-16T01:52:57.948795Z","iopub.status.idle":"2024-03-16T01:53:56.001403Z","shell.execute_reply.started":"2024-03-16T01:52:57.948765Z","shell.execute_reply":"2024-03-16T01:53:55.999747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_aids_df","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:53:56.003098Z","iopub.execute_input":"2024-03-16T01:53:56.003434Z","iopub.status.idle":"2024-03-16T01:53:56.021071Z","shell.execute_reply.started":"2024-03-16T01:53:56.003404Z","shell.execute_reply":"2024-03-16T01:53:56.019104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict","metadata":{"execution":{"iopub.status.busy":"2024-03-16T01:55:45.914006Z","iopub.execute_input":"2024-03-16T01:55:45.914400Z","iopub.status.idle":"2024-03-16T01:55:45.919660Z","shell.execute_reply.started":"2024-03-16T01:55:45.914370Z","shell.execute_reply":"2024-03-16T01:55:45.918580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_Sim = defaultdict()","metadata":{"execution":{"iopub.status.busy":"2024-03-16T02:02:52.286554Z","iopub.execute_input":"2024-03-16T02:02:52.286961Z","iopub.status.idle":"2024-03-16T02:02:52.292583Z","shell.execute_reply.started":"2024-03-16T02:02:52.286931Z","shell.execute_reply":"2024-03-16T02:02:52.291353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,row in tqdm( session_aids_df.iterrows(), total=len(session_aids_df) ):\n    \n    aid_list = row['aid']\n    \n    for item in aid_list:\n        \n        item_Sim.setdefault(item,dict())\n        \n        for related_item in aid_list:\n            if item == related_item:\n                continue\n        \n            item_Sim[item].setdefault(related_item,0)\n            \n            item_Sim[item][related_item] += 1\n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-16T02:05:37.614682Z","iopub.execute_input":"2024-03-16T02:05:37.615184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To be continued","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}