{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\nThis competition solely provides interaction data, which makes it hard to incorporate any user specific information. Based on the [notebook](https://www.kaggle.com/code/cdeotte/time-series-eda-users-and-real-sessions) from  [@cdeotte](https://www.kaggle.com/cdeotte) I try to find behavioural patterns, focusing on the first interactions so we can incorporate these information in our models.","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries and Train Data","metadata":{}},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nfrom random import sample \nimport datetime\nimport gc\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-19T22:50:10.790450Z","iopub.execute_input":"2022-11-19T22:50:10.790783Z","iopub.status.idle":"2022-11-19T22:50:11.303174Z","shell.execute_reply.started":"2022-11-19T22:50:10.790720Z","shell.execute_reply":"2022-11-19T22:50:11.302270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD TRAIN DATA. RANDOM SAMPLE 10%\ntrain = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\nsessions = list(train.session.unique())\nsample = sample(sessions,len(sessions)//10)\ntrain = train.loc[train.session.isin(sample)]\nprint('We are using random 1/10 of users. Truncated train data has shape', train.shape )\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:50:11.304617Z","iopub.execute_input":"2022-11-19T22:50:11.304870Z","iopub.status.idle":"2022-11-19T22:50:33.776343Z","shell.execute_reply.started":"2022-11-19T22:50:11.304842Z","shell.execute_reply":"2022-11-19T22:50:33.775655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MIN AND MAX TRAIN DATES\n# IF USING ORIGINAL CSV, USE \"TS * 1e6\" BELOW\ntrain.ts = pd.to_datetime(train.ts * 1e9)\nprint('Train min date and max date are:', train.ts.min(),'and', train.ts.max() )\nprint('We will truncate train data to begin Aug 1st, 2022')\ntrain = train.loc[train.ts >= pd.to_datetime('2022-08-01')]","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:50:33.777312Z","iopub.execute_input":"2022-11-19T22:50:33.777716Z","iopub.status.idle":"2022-11-19T22:50:37.759416Z","shell.execute_reply.started":"2022-11-19T22:50:33.777691Z","shell.execute_reply":"2022-11-19T22:50:37.758223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# COMPUTE DAY AND HOUR OF ACTIVITY\ntrain['day'] = train.ts.dt.day\ntrain['hour'] = train.ts.dt.hour\ntrain['month'] = train.ts.dt.month\ntrain['minutes'] = train.ts.dt.minute\ntrain['weekday'] = train.ts.dt.weekday\n\ntrain = train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:50:37.761740Z","iopub.execute_input":"2022-11-19T22:50:37.762022Z","iopub.status.idle":"2022-11-19T22:50:43.360213Z","shell.execute_reply.started":"2022-11-19T22:50:37.761999Z","shell.execute_reply":"2022-11-19T22:50:43.359141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['ts_millisec'] = train.ts.astype('int64')/1e9\ntrain.sort_values(['session','ts']).reset_index(drop=True)\n\n# THE NEXT THREE LINES REPLICATE GROUPBY DIFF\ntrain = train.sort_values(['session','ts_millisec']).reset_index(drop=True)\ntrain['d'] = train.groupby('session')['ts_millisec'].diff()\ntrain.loc[ train.session.diff()!=0, 'd'] = 0\ntrain.d = (train.d > 60*60*2).astype('int8').fillna(0)\ntrain.loc[ train.session.diff()!=0, 'd'] = 2","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:50:43.361282Z","iopub.execute_input":"2022-11-19T22:50:43.361524Z","iopub.status.idle":"2022-11-19T22:54:40.693976Z","shell.execute_reply.started":"2022-11-19T22:50:43.361503Z","shell.execute_reply":"2022-11-19T22:54:40.692682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#numbering minisessions\na =train['d']\n\nj = 0\n\nminisessions = []\n\nfor i in range(len(a)):\n    if a[i] != 0:\n        if a[i]==1:\n            j+=1\n        else: \n            j=1\n    minisessions.append(j)\ntrain['minisessions'] = minisessions\n    ","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:54:40.957543Z","iopub.execute_input":"2022-11-19T22:54:40.958299Z","iopub.status.idle":"2022-11-19T22:56:54.520131Z","shell.execute_reply.started":"2022-11-19T22:54:40.958272Z","shell.execute_reply":"2022-11-19T22:56:54.519055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['clicks']=0\ntrain['carts']=0\ntrain['orders']=0\n\ntrain.loc[train.type ==0, 'clicks'] =1\ntrain.loc[train.type ==1, 'carts'] =1\ntrain.loc[train.type ==2, 'orders'] =1\n\n#create a new dataframe where minisessions are grouped\ntrain_minisession = train.groupby(['session', 'minisessions']).agg({'hour': 'min', 'weekday': 'min', 'clicks': 'sum', \n                                                'carts': 'sum', 'orders':'sum'\n                                               }).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:56:54.521527Z","iopub.execute_input":"2022-11-19T22:56:54.521886Z","iopub.status.idle":"2022-11-19T22:56:55.064698Z","shell.execute_reply.started":"2022-11-19T22:56:54.521853Z","shell.execute_reply":"2022-11-19T22:56:55.063515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for further analysis\n# train_minisession['conversion_rate']= train_minisession['orders']/train_minisession['clicks']\n# train_minisession['conversion_rate'] = train_minisession['conversion_rate'].fillna(0)\n# train_minisession['conversion_rate'] = train_minisession['conversion_rate'].replace(np.inf, 0)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:56:59.645822Z","iopub.execute_input":"2022-11-19T22:56:59.646118Z","iopub.status.idle":"2022-11-19T22:56:59.698620Z","shell.execute_reply.started":"2022-11-19T22:56:59.646094Z","shell.execute_reply":"2022-11-19T22:56:59.697584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Minisessions and Weekdays","metadata":{}},{"cell_type":"markdown","source":"Users are seperated by number of minisessions (threshold = 20) and on which day they had their first minisession. Sessions are plotted against weekdays in order to see if users chose same days for interaction.","metadata":{}},{"cell_type":"code","source":"weekdays = {0: 'Monday', 1: 'Tuesday', 2: 'Wednesday', 3: 'Thursday', 4: 'Friday', 5:'Saturday', 6: 'Sunday'}\n\ntrain_minisession['order_bool_ms'] = 0\ntrain_minisession.loc[train_minisession.orders != 0, 'order_bool_ms'] = train_minisession['minisessions'] #facilitates the plot\n\ntmp = train_minisession.groupby('session')['minisessions'].count()\n\nfor j in range(2):\n    tmp = train_minisession.groupby('session')['minisessions'].count()\n    tmp = tmp[tmp>2] #discarding users with less than 3 sessions\n    if j == 0: tmp = tmp[tmp<20]\n    else: tmp = tmp[tmp>=20]\n    tmp = tmp.keys()\n    train_ms_copy = train_minisession.copy()\n    train_ms_copy = train_ms_copy.set_index('session')\n    train_ms_copy = train_ms_copy.loc[tmp,:]\n    train_ms_copy = train_ms_copy.reset_index()\n    for i in range(7):\n        sessions = train_ms_copy.loc[(train_ms_copy.weekday == i)& (train_ms_copy.minisessions == 1), 'session'].unique() \n        sessions = sessions[:200]\n        plt.figure(figsize=(20,5))\n        for session in sessions:\n            x = train_ms_copy.loc[(train_ms_copy.session == session), 'minisessions']\n            y = train_ms_copy.loc[(train_ms_copy.session == session), 'weekday']\n            plt.plot(x,y)  \n            if j == 0: plt.title(f'First session on {weekdays[i]} -Few Session User')\n            else: plt.title(f'First session on {weekdays[i]} -Many Session User')\n            x = train_ms_copy.loc[(train_ms_copy.session == session), 'order_bool_ms']\n            if session == sessions[0]: plt.plot(x,y, 'r^', markevery = x>0, ms = 15, label = 'orders')\n            else: plt.plot(x,y, 'r^', markevery = x>0, ms = 15)\n            plt.xlabel('Minisession no')\n            plt.ylabel('Weekdays')\n            plt.legend(loc = 'upper right')\n        plt.show()\n    del train_ms_copy","metadata":{"execution":{"iopub.status.busy":"2022-11-19T23:13:48.300427Z","iopub.execute_input":"2022-11-19T23:13:48.300816Z","iopub.status.idle":"2022-11-19T23:14:29.192823Z","shell.execute_reply.started":"2022-11-19T23:13:48.300791Z","shell.execute_reply":"2022-11-19T23:14:29.191984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Observations\n- There seems to be no clear pattern with regard to user being active on the same days\n- It seems the earlier in the week the user begins his sessions, the more sessions will he have","metadata":{}},{"cell_type":"markdown","source":"# How do clicks, carts, orders evolve during minisessions?","metadata":{}},{"cell_type":"markdown","source":"Overall","metadata":{}},{"cell_type":"code","source":"ms_pivot = pd.pivot_table(train_minisession, columns='minisessions')\nx = ms_pivot.columns[:10]\ny_clicks = ms_pivot.loc['clicks', :10]\ny_carts = ms_pivot.loc['carts', :10]\ny_orders = ms_pivot.loc['orders', :10]\nplt.figure(figsize = (20,10))\nplt.plot(x,y_clicks, x, y_carts, x, y_orders)\nplt.legend(['clicks', 'carts', 'orders'])\nplt.title('# of actions (mean) per minisession')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-19T23:31:00.231434Z","iopub.execute_input":"2022-11-19T23:31:00.231818Z","iopub.status.idle":"2022-11-19T23:31:00.646108Z","shell.execute_reply.started":"2022-11-19T23:31:00.231793Z","shell.execute_reply":"2022-11-19T23:31:00.644594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now seperating by day of the frst session","metadata":{}},{"cell_type":"code","source":"\nfor i in range(7):\n    sessions = train_minisession.loc[(train_minisession.weekday == i)& (train_minisession.minisessions == 1), 'session'].unique() \n    sessions = sessions[:200]\n    plt.figure(figsize=(20,5))\n\n    train_minisession = train_minisession.set_index('session')\n    ms_pivot = train_minisession.loc[sessions, :]\n    train_minisession = train_minisession.reset_index()\n    ms_pivot = pd.pivot_table(ms_pivot, columns='minisessions')\n    x = ms_pivot.columns[:10]\n    y_clicks = ms_pivot.loc['clicks', :10]\n    y_carts = ms_pivot.loc['carts', :10]\n    y_orders = ms_pivot.loc['orders', :10]\n    plt.figure(figsize = (20,10))\n    plt.plot(x,y_clicks, x, y_carts, x, y_orders)\n    plt.legend(['clicks', 'carts', 'orders'])\n    plt.title(f'# of actions (mean) per minisession, first session on {weekdays[i]}')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-11-19T23:47:54.480531Z","iopub.execute_input":"2022-11-19T23:47:54.480893Z","iopub.status.idle":"2022-11-19T23:48:00.816792Z","shell.execute_reply.started":"2022-11-19T23:47:54.480869Z","shell.execute_reply":"2022-11-19T23:48:00.815656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see different behaviour on beginning vs end of the week.","metadata":{}},{"cell_type":"markdown","source":"Ideas on what to analyse further\n- correlation with popular products (lags) \n- how many clicks until a product is ordered\n- invovle time dimension (night vs day)","metadata":{}}]}