{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Idea\nThis notebook aims to understand the correlation between clicks and carts/order within the same session. In particular, I'm going to check how many times a cart/order belongs to the `aid` clicked by the user in the session. This analysis could be useful in determining whether we should focus mainly or partially on already clicked items when predicting carts and orders.","metadata":{}},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy.sparse as sps\nimport gc\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport plotly.graph_objects as go","metadata":{"execution":{"iopub.status.busy":"2022-12-05T19:36:32.106187Z","iopub.execute_input":"2022-12-05T19:36:32.106666Z","iopub.status.idle":"2022-12-05T19:36:32.205379Z","shell.execute_reply.started":"2022-12-05T19:36:32.106574Z","shell.execute_reply":"2022-12-05T19:36:32.204444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T19:36:32.207081Z","iopub.execute_input":"2022-12-05T19:36:32.207503Z","iopub.status.idle":"2022-12-05T19:36:50.512599Z","shell.execute_reply.started":"2022-12-05T19:36:32.207472Z","shell.execute_reply":"2022-12-05T19:36:50.511696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of times a cart/order `aid` is also a click in the same session","metadata":{}},{"cell_type":"code","source":"train_sessions = train.session.unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T19:36:50.513749Z","iopub.execute_input":"2022-12-05T19:36:50.514500Z","iopub.status.idle":"2022-12-05T19:36:52.364057Z","shell.execute_reply.started":"2022-12-05T19:36:50.514457Z","shell.execute_reply":"2022-12-05T19:36:52.362412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks = train[(train.type == 0)].groupby('session').apply(lambda x: list(x['aid']))\ncarts = train[(train.type == 1)].groupby('session').apply(lambda x: list(x['aid']))\norders = train[(train.type == 2)].groupby('session').apply(lambda x: list(x['aid']))","metadata":{"execution":{"iopub.status.busy":"2022-12-05T19:36:52.366547Z","iopub.execute_input":"2022-12-05T19:36:52.366916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'session': train_sessions})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.merge(clicks.rename('clicks'), how='left', left_on='session', right_index=True)\ndf = df.merge(carts.rename('carts'), how='left', left_on='session', right_index=True)\ndf = df.merge(orders.rename('orders'), how='left', left_on='session', right_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del clicks, carts, orders","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Count the number of _carts/orders_ `aid`s that are also _clicks_","metadata":{}},{"cell_type":"code","source":"carts_in_clicks = []\norders_in_clicks = []\nfor clks, crts, ords in tqdm(zip(df.clicks, df.carts, df.orders)):\n    if isinstance(crts, list):\n        crts_len = len(crts)\n        crts_in = sum([x in clks for x in crts])\n        cart_res = crts_in / crts_len\n        carts_in_clicks.append(cart_res)\n    \n    if isinstance(ords, list):\n        ords_len = len(ords)\n        ords_in = sum([x in clks for x in ords])\n        ord_res = ords_in / ords_len\n        orders_in_clicks.append(ord_res)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Carts in Clicks","metadata":{}},{"cell_type":"code","source":"res = sum(carts_in_clicks) / len(carts_in_clicks)\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['Cart-In-Clicks', 'Cart Not In-Clicks'],\n            orientation='h'))\nfig.update_layout(title='Carts in clicks')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Orders in Clicks","metadata":{}},{"cell_type":"code","source":"res = sum(orders_in_clicks) / len(orders_in_clicks)\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['Order-In-Clicks', 'Order Not In-Clicks'],\n            orientation='h'))\nfig.update_layout(title='Orders in clicks')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T07:29:36.792535Z","iopub.execute_input":"2022-11-30T07:29:36.793587Z","iopub.status.idle":"2022-11-30T07:29:36.810453Z","shell.execute_reply.started":"2022-11-30T07:29:36.793547Z","shell.execute_reply":"2022-11-30T07:29:36.808769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Even if it could be less informative, we could also check the number of times a _cart_ `aid` is in the set of _orders_, and viceversa","metadata":{}},{"cell_type":"code","source":"# Check also orders_in_carts and carts_in_orders\ncarts_in_orders = []\norders_in_carts = []\nfor crts, ords in tqdm(zip(df.carts, df.orders)):\n    if isinstance(crts, list) and isinstance(ords, list):\n        # Carts in orders\n        crts_len = len(crts)\n        crts_in = sum([x in ords for x in crts])\n        cart_res = crts_in / crts_len\n        # Orders in carts\n        ords_len = len(ords)\n        ords_in = sum([x in crts for x in ords])\n        ord_res = ords_in / ords_len\n        carts_in_orders.append(cart_res)\n        orders_in_carts.append(ord_res)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Carts in Orders","metadata":{}},{"cell_type":"code","source":"res = sum(carts_in_orders) / len(carts_in_orders)\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['Cart In Order', 'Cart Not In Order'],\n            orientation='h'))\nfig.update_layout(title='Carts in Orders')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Orders in Carts","metadata":{}},{"cell_type":"code","source":"res = sum(orders_in_carts) / len(orders_in_carts)\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['Order in Cart', 'Order Not In Cart'],\n            orientation='h'))\nfig.update_layout(title='Orders in Carts')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df, carts_in_orders, orders_in_carts, carts_in_clicks, orders_in_clicks","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Note\nAs you can see, almost all the items in _carts/orders_ belongs to the `aid` clicked by the user. However, this approach doesn't take into account the temporal precedence of different event types. \n\nA more effective strategy to show if a user clicks an item before adding it to carts/orders is to consider the order of event types within the session. Thus, we need to check whether a __cart/order__ event on some `aid` follows a __click__ event on the same `aid`\n","metadata":{}},{"cell_type":"markdown","source":"# Temporal Evaluation\nUsing Pandas is extremely slow for such a computation and we are likely to run out of memory. For this reason, I'm going to use Cython to speed-up the task.","metadata":{}},{"cell_type":"code","source":"%load_ext Cython","metadata":{"execution":{"iopub.status.busy":"2022-11-30T08:26:24.006851Z","iopub.execute_input":"2022-11-30T08:26:24.007759Z","iopub.status.idle":"2022-11-30T08:26:24.744461Z","shell.execute_reply.started":"2022-11-30T08:26:24.007717Z","shell.execute_reply":"2022-11-30T08:26:24.743424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython\n\nimport cython\ncimport cython\ncimport numpy as np\nimport numpy as np\nfrom tqdm import tqdm, trange\n\nctypedef np.int32_t INT32_t\nctypedef np.uint8_t UINT8_t\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\ncpdef (float, float) temporal_in_clicks(INT32_t[:] sessions, INT32_t[:] aids, UINT8_t[:] types):\n    shape = len(sessions)\n    # Global variables\n    cdef int n_sess_with_carts = 0\n    cdef int n_sess_with_orders = 0\n    cdef float n_carts_in_clicks = 0.0\n    cdef float n_orders_in_clicks = 0.0\n    # Per-session variables\n    cdef int n_sess_carts = 0\n    cdef int n_sess_orders = 0\n    cdef int n_sess_carts_in_previous_clicks = 0\n    cdef int n_sess_orders_in_previous_clicks = 0\n    cdef list clicks = []\n    # Loop variables\n    cdef INT32_t curr_sess # Define the current session\n    cdef INT32_t s # Define the current event session\n    cdef UINT8_t t # Define the current event type\n    cdef INT32_t item # Define the current item\n    cdef int i = 0 # Loop index\n    \n    for i in trange(shape):\n        s = sessions[i]\n        t = types[i]\n        item = aids[i]\n        # Reset Variables if session changed\n        if s != curr_sess:\n            # Save previous session results\n            if n_sess_carts > 0:\n                n_sess_with_carts += 1\n                n_carts_in_clicks += n_sess_carts_in_previous_clicks / n_sess_carts\n            if n_sess_orders > 0:\n                n_sess_with_orders += 1\n                n_orders_in_clicks += n_sess_orders_in_previous_clicks / n_sess_orders\n            # Reset session variables\n            clicks = []\n            n_sess_carts = 0\n            n_sess_orders = 0\n            n_sess_carts_in_previous_clicks = 0\n            n_sess_orders_in_previous_clicks = 0\n            # Update current session\n            curr_sess = s\n\n        if t == 0:\n            clicks.append(item)\n            continue\n        if t == 1:\n            n_sess_carts += 1\n            n_sess_carts_in_previous_clicks += item in clicks\n        if t == 2:\n            n_sess_orders += 1\n            n_sess_orders_in_previous_clicks += item in clicks\n    return (n_carts_in_clicks / n_sess_with_carts, n_orders_in_clicks / n_sess_with_orders)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T08:26:26.967857Z","iopub.execute_input":"2022-11-30T08:26:26.968765Z","iopub.status.idle":"2022-11-30T08:26:35.779175Z","shell.execute_reply.started":"2022-11-30T08:26:26.968724Z","shell.execute_reply":"2022-11-30T08:26:35.777389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"carts_in_clicks, orders_in_click = temporal_in_clicks(train.session.values, train.aid.values, train.type.values)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T08:26:35.78213Z","iopub.execute_input":"2022-11-30T08:26:35.782726Z","iopub.status.idle":"2022-11-30T08:27:43.684433Z","shell.execute_reply.started":"2022-11-30T08:26:35.782677Z","shell.execute_reply":"2022-11-30T08:27:43.683424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = carts_in_clicks\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['In-Clicks', 'Not In-Clicks'],\n            orientation='h'))\nfig.update_layout(title='Carts in clicks')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T08:28:00.851044Z","iopub.execute_input":"2022-11-30T08:28:00.851476Z","iopub.status.idle":"2022-11-30T08:28:01.243637Z","shell.execute_reply.started":"2022-11-30T08:28:00.851442Z","shell.execute_reply":"2022-11-30T08:28:01.242332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = orders_in_click\nfig = go.Figure(go.Bar(\n            x=[res, 1 - res],\n            y=['In-Clicks', 'Not In-Clicks'],\n            orientation='h'))\nfig.update_layout(title='Orders in Clicks')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\nAs we can see, most of the items in _carts_ and _orders_ are preceded by a __click__ on the same item. Does it mean our focus should mostly be on already clicked items?","metadata":{}}]}