{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\n\nimport numpy as np\nimport pandas as pd\n\nfrom matplotlib import dates\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks = pd.read_json('/kaggle/input/otto-recommender-system/train.jsonl', lines=True, chunksize=100_000)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.DataFrame()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data loading logic taken from Charlie Crane's notebook here: (https://www.kaggle.com/code/crained/otto-rename-columns)","metadata":{}},{"cell_type":"code","source":"for i, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid':     [],\n        'ts':      [],\n        'type':    []\n    }\n\n    if i >= 2:\n        break\n    for session, events in zip(chunk['session'].tolist(), \\\n                               chunk['events'].tolist()):\n        for event in events:\n            event_dict['session'].append(session)\n            event_dict['aid'].append(event['aid'])\n            event_dict['ts'].append(event['ts'])\n            event_dict['type'].append(event['type'])\n    chunk_session = pd.DataFrame(event_dict)\n    train = pd.concat([train, chunk_session])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.rename(index=str, columns={'session': 'customer_id',\n                                 'aid' : 'product_code',\n                                 'ts' : 'time_stamp',\n                                 'type' : 'event_type'}, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['time_stamp'] = pd.to_datetime(train['time_stamp'], unit='ms')\ndisplay(train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Customer session temporal visualisation (randomly selected)","metadata":{}},{"cell_type":"code","source":"for customer in random.sample(range(200000), 10):\n    customer_df = train[train['customer_id'] == customer]\n    customer_df = customer_df.sort_values(by='time_stamp')\n    fig, ax = plt.subplots(figsize=(15, 5))\n    ax.set_title('Customer Session Overview')\n    sns.scatterplot(x='time_stamp', y='event_type', data=customer_df, hue='event_type', ax=ax)\n    plt.show()    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Event dependency percentages\n\n- In what fraction of instances an order event happens after add to cart (0.63%)\n- In what fraction of instances an add to cart event occurs after a click event (5.89%)\n- In what fraction of instances a click event occurs after a click event (83.64%)\n\nNote: these percentages are calculated across the entire dataset and does not take into consideration a user session. ","metadata":{}},{"cell_type":"markdown","source":"### Order after add to cart","metadata":{}},{"cell_type":"code","source":"# Percentage of orders occuring after add to cart events\ntrain['order'] = train['event_type'].apply(lambda x: 1 if x == 'orders' else 0)\ntrain['add_to_cart'] = train['event_type'].apply(lambda x: 1 if x == 'carts' else 0)\ntrain['order_after_cart'] = train['order'].shift(-1) + train['add_to_cart']\ntrain['order_after_cart'] = train['order_after_cart'].apply(lambda x: 1 if x == 2 else 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perc = train['order_after_cart'].value_counts(normalize=True)[1]\n\nprint(f'Percentage of orders occuring after add to cart events: {perc:.2%}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add to cart after click","metadata":{}},{"cell_type":"code","source":"# Percentage of add_to_cart events occuring after click events\ntrain['click'] = train['event_type'].apply(lambda x: 1 if x == 'clicks' else 0)\ntrain['cart_after_click'] = train['click'].shift(-1) + train['add_to_cart']\ntrain['cart_after_click'] = train['cart_after_click'].apply(lambda x: 1 if x == 2 else 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perc = train['cart_after_click'].value_counts(normalize=True)[1]\n\nprint(f'Percentage of add_to_cart events occuring after click events: {perc:.2%}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Click after click","metadata":{}},{"cell_type":"code","source":"# Percentage of click events occuring after click events\ntrain['click_after_click'] = train['click'].shift(-1) + train['click']\ntrain['click_after_click'] = train['click_after_click'].apply(lambda x: 1 if x == 2 else 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perc = train['click_after_click'].value_counts(normalize=True)[1]\n\nprint(f'Percentage of click events occuring after click events: {perc:.2%}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train = train.groupby('customer_id').count()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(grouped_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Per customer statistics\n\nFrom the selected subset of 200k customers, on average we see a user clicks, add to cart or order around 51 products with a standard deviation of 75. Maximum number of events in a customer session is 495 and minimum session is 2, indicating potentially a long tail of customer sessions with relatively large number of events. ","metadata":{}},{"cell_type":"code","source":"grouped_train.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of number of products viewed by customers","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.distplot(grouped_train['product_code'], kde=False, bins=100)\nplt.title('Distribution of number of products viewed by customer')\nplt.xlabel('Number of products viewed')\nplt.ylabel('Number of customers')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Event distribution across entire userbase\n\nMajority of the events in the sampled data are clicks, followed by addition to carts and orders. It is evident that a very small chunk of customers sessions translate to a conversion. At this point it will be interesting to understand the relationship b/w length of the session and the conversion rate.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.histplot(train['event_type'], stat='percent', kde=False, bins=100)\nplt.title('Distribution of event types')\nplt.xlabel('Event type')\nplt.ylabel('Number of events')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train_orders = train[train['event_type'] == 'orders'].groupby('customer_id').count().join(grouped_train, lsuffix='_orders', rsuffix='_views')   \ngrouped_train_orders = grouped_train_orders.drop(['product_code_views', 'time_stamp_views'], axis=1).rename(columns={'event_type_views': 'event_count'})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(grouped_train_orders)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of products viewed vs. number of products ordered\n\nThere seems to be a slight correlation b/w number of products viewed vs number of products ordered by a customer. ","metadata":{}},{"cell_type":"code","source":"# plot trend between event count and event type orders\nplt.figure(figsize=(10, 5))\nsns.scatterplot(x=grouped_train_orders['event_count'], y=grouped_train_orders['product_code_orders'])\nplt.title('Distribution of number of products ordered by customer')\nplt.xlabel('Number of products viewed')\nplt.ylabel('Number of products ordered')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display( grouped_train_orders[grouped_train_orders['product_code_orders'] < 100])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of number of products ordered by customers","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.histplot(grouped_train_orders['product_code_orders'], stat='percent', kde=False, bins=100)\nplt.title('Distribution of number of products ordered by customer')\nplt.xlabel('Number of products ordered')\nplt.ylabel('Number of customers')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train_clicks = train[train['event_type'] == 'clicks'].groupby('customer_id').count().join(grouped_train, lsuffix='_clicks', rsuffix='_views')   \ngrouped_train_clicks = grouped_train_clicks.drop(['product_code_views', 'time_stamp_views'], axis=1).rename(columns={'event_type_views': 'event_count'})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(grouped_train_clicks)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of number of products clicked","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.histplot(grouped_train_clicks['product_code_clicks'], stat='percent', kde=False, bins=100)\nplt.title('Distribution of number of products clicked by customer')\nplt.xlabel('Number of products clicked')\nplt.ylabel('Number of customers')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_train_carts = train[train['event_type'] == 'carts'].groupby('customer_id').count().join(grouped_train, lsuffix='_carts', rsuffix='_views')\ngrouped_train_carts = grouped_train_carts.drop(['product_code_views', 'time_stamp_views'], axis=1).rename(columns={'event_type_views': 'event_count'})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(grouped_train_carts)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of number of products added to cart","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nsns.histplot(grouped_train_carts['product_code_carts'], stat='percent', kde=False, bins=100)\nplt.title('Distribution of number of products added to cart by customer')\nplt.xlabel('Number of products added to cart')\nplt.ylabel('Number of customers')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of products ordered vs. number of products clicked","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nsns.barplot(x=grouped_train_orders['product_code_orders'], y=grouped_train_clicks['product_code_clicks'])\nplt.title('Distribution of number of products ordered by customer')\nplt.xlabel('Number of products ordered')\nplt.ylabel('Number of products clicked')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of products ordered vs. number of products added to cart","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nsns.barplot(x=grouped_train_orders['product_code_orders'], y=grouped_train_carts['product_code_carts'])\nplt.title('Distribution of number of products ordered by customer')\nplt.xlabel('Number of products ordered')\nplt.ylabel('Number of products added to carts')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}