{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"### Imports ###\nimport multiprocessing\nimport time\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport os\nimport random\nimport numpy as np\nimport json\nfrom datetime import timedelta\nfrom collections import Counter\nfrom tqdm.notebook import tqdm\nfrom heapq import nlargest\nfrom typing import List, Dict, Union, Set\nimport plotly.express as px\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_theme()\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-09T09:39:40.973792Z","iopub.execute_input":"2022-11-09T09:39:40.974255Z","iopub.status.idle":"2022-11-09T09:39:42.182495Z","shell.execute_reply.started":"2022-11-09T09:39:40.974155Z","shell.execute_reply":"2022-11-09T09:39:42.181320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Paths ###\n\nDATA_PATH = Path('/kaggle/input/otto-recommender-system/')\nTRAIN_PATH = DATA_PATH/'train.jsonl'\nTEST_PATH = DATA_PATH/'test.jsonl'\nSAMPLE_SUB_PATH = Path('../input/otto-recommender-system/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:39:42.185739Z","iopub.execute_input":"2022-11-09T09:39:42.186260Z","iopub.status.idle":"2022-11-09T09:39:42.193826Z","shell.execute_reply.started":"2022-11-09T09:39:42.186215Z","shell.execute_reply":"2022-11-09T09:39:42.191815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code","metadata":{}},{"cell_type":"code","source":"def compute_events_set(events: Union[List[Dict],pd.core.series.Series]) -> Set[List]:\n    event_dict = {'clicks':[], 'carts':[], 'orders':[]}\n    for d in events:\n        event_dict[d['type']].append(d['aid'])\n    events_set = {k: set(v) for k,v in event_dict.items()}\n    return events_set\n\ndef compute_cond_probs(events_set: Set[List], roundoff=2) -> Dict:\n    cond_probs = {}\n    # conditional on clicks\n    if events_set['clicks']:\n        cond_probs[\"P(carts|clicks)\"] = len(events_set['carts'] & events_set['clicks'])/len(events_set['clicks'])\n        cond_probs[\"P(orders|clicks)\"] = len(events_set['orders'] & events_set['clicks'])/len(events_set['clicks'])\n    # conditional on carts\n    if events_set['carts']:\n        cond_probs[\"P(clicks|carts)\"] = len(events_set['clicks'] & events_set['carts'])/len(events_set['carts'])\n        cond_probs[\"P(orders|carts)\"] = len(events_set['orders'] & events_set['carts'])/len(events_set['carts'])    \n    # conditional on orders\n    if events_set['orders']:\n        cond_probs[\"P(clicks|orders)\"] = len(events_set['clicks'] & events_set['orders'])/len(events_set['orders'])\n        cond_probs[\"P(carts|orders)\"] = len(events_set['carts'] & events_set['orders'])/len(events_set['orders'])\n        \n    return {k:round(v,roundoff) for k,v in cond_probs.items()} if roundoff else cond_probs\n\ndef prob_single_session(events: Union[List[Dict],pd.core.series.Series]) -> Dict:\n    events_set = compute_events_set(events)\n    cond_probs = compute_cond_probs(events_set)\n    return cond_probs\n\ndef prob_many_sessions(df):\n    sessions_cond_probs = [{\"session\":sid,**prob_single_session(events)} for sid, events in zip(df.session, df.events)]\n    return sessions_cond_probs","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:39:42.196061Z","iopub.execute_input":"2022-11-09T09:39:42.196557Z","iopub.status.idle":"2022-11-09T09:39:42.212920Z","shell.execute_reply.started":"2022-11-09T09:39:42.196512Z","shell.execute_reply":"2022-11-09T09:39:42.211547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 10_000\nall_probs = []\n\nchunks = pd.read_json(TRAIN_PATH, lines=True, chunksize=sample_size)\n\nfor idx, chunk in enumerate(tqdm(chunks)):\n    all_probs.extend(prob_many_sessions(chunk))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:39:42.247691Z","iopub.execute_input":"2022-11-09T09:39:42.249152Z","iopub.status.idle":"2022-11-09T09:56:06.484428Z","shell.execute_reply.started":"2022-11-09T09:39:42.249086Z","shell.execute_reply":"2022-11-09T09:56:06.482568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond_probs_df = pd.DataFrame(all_probs)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T09:57:16.040772Z","iopub.execute_input":"2022-11-09T09:57:16.041225Z","iopub.status.idle":"2022-11-09T09:57:48.033917Z","shell.execute_reply.started":"2022-11-09T09:57:16.041180Z","shell.execute_reply":"2022-11-09T09:57:48.032440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond_probs_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:26:13.952309Z","iopub.execute_input":"2022-11-09T10:26:13.952744Z","iopub.status.idle":"2022-11-09T10:26:13.971991Z","shell.execute_reply.started":"2022-11-09T10:26:13.952708Z","shell.execute_reply":"2022-11-09T10:26:13.970509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond_probs_df.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond_probs_df.set_index('session').corr()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:43:10.310943Z","iopub.execute_input":"2022-11-09T10:43:10.311736Z","iopub.status.idle":"2022-11-09T10:43:12.656715Z","shell.execute_reply.started":"2022-11-09T10:43:10.311695Z","shell.execute_reply":"2022-11-09T10:43:12.655502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cond_probs_df.set_index('session').corr(), annot=True, fmt=\".2f\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:47:35.289247Z","iopub.execute_input":"2022-11-09T10:47:35.289828Z","iopub.status.idle":"2022-11-09T10:47:38.009803Z","shell.execute_reply.started":"2022-11-09T10:47:35.289780Z","shell.execute_reply":"2022-11-09T10:47:38.008700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(cond_probs_df, x=\"P(carts|clicks)\", nbins=10, histnorm=\"percent\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:24:32.213930Z","iopub.execute_input":"2022-11-10T14:24:32.214421Z","iopub.status.idle":"2022-11-10T14:24:32.243749Z","shell.execute_reply.started":"2022-11-10T14:24:32.214326Z","shell.execute_reply":"2022-11-10T14:24:32.242486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(cond_probs_df, x=\"P(orders|clicks)\", nbins=10, histnorm=\"percent\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:18:20.642919Z","iopub.execute_input":"2022-11-09T10:18:20.643671Z","iopub.status.idle":"2022-11-09T10:18:23.775630Z","shell.execute_reply.started":"2022-11-09T10:18:20.643595Z","shell.execute_reply":"2022-11-09T10:18:23.772194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.histogram(cond_probs_df, x=\"P(clicks|carts)\", nbins=10, histnorm=\"percent\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:25:06.151595Z","iopub.execute_input":"2022-11-09T10:25:06.152167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.histogram(cond_probs_df, x=\"P(orders|carts)\", nbins=10, histnorm=\"percent\")\n# fig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.histogram(cond_probs_df, x=\"P(carts|orders)\", nbins=10, histnorm=\"percent\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:15:55.805433Z","iopub.execute_input":"2022-11-09T10:15:55.805913Z","iopub.status.idle":"2022-11-09T10:15:58.884692Z","shell.execute_reply.started":"2022-11-09T10:15:55.805878Z","shell.execute_reply":"2022-11-09T10:15:58.881829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = px.histogram(cond_probs_df, x=\"P(clicks|orders)\", nbins=10, histnorm=\"percent\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T10:14:26.893205Z","iopub.execute_input":"2022-11-09T10:14:26.893687Z","iopub.status.idle":"2022-11-09T10:14:29.936313Z","shell.execute_reply.started":"2022-11-09T10:14:26.893652Z","shell.execute_reply":"2022-11-09T10:14:29.933989Z"},"trusted":true},"execution_count":null,"outputs":[]}]}