{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OTTO - EDA + Basic LSTM Model\n\nThe Load-in is based on the fantastic notebook: https://www.kaggle.com/code/columbia2131/otto-fast-dataframe-loading-in-parquet-format","metadata":{}},{"cell_type":"code","source":"import os \n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom pathlib import Path\nfrom glob import glob\nfrom tqdm import tqdm\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:16.240714Z","iopub.execute_input":"2022-11-05T11:09:16.241649Z","iopub.status.idle":"2022-11-05T11:09:22.661471Z","shell.execute_reply.started":"2022-11-05T11:09:16.241530Z","shell.execute_reply":"2022-11-05T11:09:22.660373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfiles = sorted(glob('../input/otto-chunk-data-inparquet-format/train_parquet/*'))[:5]\ndfs = []\n\nfor path in tqdm(files):\n    dfs.append(pd.read_parquet(path))\n\ndfs = pd.concat(dfs).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:22.665359Z","iopub.execute_input":"2022-11-05T11:09:22.665884Z","iopub.status.idle":"2022-11-05T11:09:28.988690Z","shell.execute_reply.started":"2022-11-05T11:09:22.665854Z","shell.execute_reply":"2022-11-05T11:09:28.987233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:28.990509Z","iopub.execute_input":"2022-11-05T11:09:28.990910Z","iopub.status.idle":"2022-11-05T11:09:29.011542Z","shell.execute_reply.started":"2022-11-05T11:09:28.990867Z","shell.execute_reply":"2022-11-05T11:09:29.010572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic EDA","metadata":{}},{"cell_type":"markdown","source":"## Let convert the timestamps to readable time","metadata":{}},{"cell_type":"code","source":"dfs['ts_conv'] = pd.to_datetime(dfs['ts'], unit='ms')","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:29.014442Z","iopub.execute_input":"2022-11-05T11:09:29.014859Z","iopub.status.idle":"2022-11-05T11:09:29.451655Z","shell.execute_reply.started":"2022-11-05T11:09:29.014819Z","shell.execute_reply":"2022-11-05T11:09:29.450515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets see which time we are looking at!","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:22:11.560940Z","iopub.execute_input":"2022-11-02T11:22:11.561300Z","iopub.status.idle":"2022-11-02T11:22:11.566180Z","shell.execute_reply.started":"2022-11-02T11:22:11.561270Z","shell.execute_reply":"2022-11-02T11:22:11.565243Z"}}},{"cell_type":"code","source":"print(f'Years: {dfs[\"ts_conv\"].dt.year.unique()}, \\n Months:  {dfs[\"ts_conv\"].dt.month.unique()} \\n Days: {dfs[\"ts_conv\"].dt.day.unique()}  ')","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:29.453339Z","iopub.execute_input":"2022-11-05T11:09:29.453875Z","iopub.status.idle":"2022-11-05T11:09:35.480287Z","shell.execute_reply.started":"2022-11-05T11:09:29.453828Z","shell.execute_reply":"2022-11-05T11:09:35.479026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**So we have data from the 31.07.2022 to 28.08.2022**","metadata":{}},{"cell_type":"code","source":"print(f'Minimum timestep: {dfs[\"ts_conv\"].min()}, Maximum timestep {dfs[\"ts_conv\"].max()}')  ","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:35.481831Z","iopub.execute_input":"2022-11-05T11:09:35.482437Z","iopub.status.idle":"2022-11-05T11:09:35.605860Z","shell.execute_reply.started":"2022-11-05T11:09:35.482396Z","shell.execute_reply":"2022-11-05T11:09:35.604703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lets look at the Session lengths","metadata":{}},{"cell_type":"code","source":"dfs_grouped = dfs.groupby('session')['ts_conv'].agg([min,max])\ndfs_grouped['diff'] = dfs_grouped['max']-dfs_grouped['min']","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:35.607582Z","iopub.execute_input":"2022-11-05T11:09:35.607966Z","iopub.status.idle":"2022-11-05T11:09:36.297839Z","shell.execute_reply.started":"2022-11-05T11:09:35.607928Z","shell.execute_reply":"2022-11-05T11:09:36.296756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting the Session length in days ","metadata":{"execution":{"iopub.status.busy":"2022-11-02T11:40:04.846271Z","iopub.execute_input":"2022-11-02T11:40:04.846636Z","iopub.status.idle":"2022-11-02T11:40:04.851166Z","shell.execute_reply.started":"2022-11-02T11:40:04.846603Z","shell.execute_reply":"2022-11-02T11:40:04.850287Z"}}},{"cell_type":"code","source":"dfs_grouped['diff'].dt.days.sort_values().value_counts(sort=False).plot(kind='bar', figsize=(10,8), title='Distribution of Session Lengths')\nplt.xlabel('Session Length [Days]')\nplt.ylabel('Count of Sessions')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:36.299364Z","iopub.execute_input":"2022-11-05T11:09:36.300298Z","iopub.status.idle":"2022-11-05T11:09:36.860645Z","shell.execute_reply.started":"2022-11-05T11:09:36.300255Z","shell.execute_reply":"2022-11-05T11:09:36.859484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That is a very peculiar distribution. The two highest counts are for the lowest and highest Session length. Lets examine what is going on!","metadata":{}},{"cell_type":"code","source":"idx_0 = dfs_grouped.query('diff.dt.days == 0').index.to_list()\nidx_27 = dfs_grouped.query('diff.dt.days == 27').index.to_list()\n","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:36.861985Z","iopub.execute_input":"2022-11-05T11:09:36.862874Z","iopub.status.idle":"2022-11-05T11:09:36.915520Z","shell.execute_reply.started":"2022-11-05T11:09:36.862836Z","shell.execute_reply":"2022-11-05T11:09:36.914530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['tab:olive', 'tab:orange', 'tab:red']\n\nfig, axs = plt.subplots(3, 2, figsize=(15,20))\ndfs.query('session in @idx_0').groupby('session').size().plot.hist(ax=axs[1,0])\ndfs.query('session in @idx_0').type.value_counts(normalize=True).plot(kind='bar',ax=axs[2,0], color=colors)\ndfs.query('session in @idx_0').groupby('session').size().plot(ax=axs[0,0])\ndfs.query('session in @idx_27').type.value_counts(normalize=True).plot(kind='bar',ax=axs[2,1],  color=colors)\ndfs.query('session in @idx_27').groupby('session').size().plot(ax=axs[0,1], sharey = axs[0,0])\ndfs.query('session in @idx_27').groupby('session').size().plot.hist(ax=axs[1,1])\naxs[0,0].set_ylabel('Count of Actions')\naxs[1,0].set_ylabel('Normalized Count')\naxs[0,0].set_title('Actions per Session (Session Length = 0 days)')\naxs[0,1].set_title('Actions per Session (Session Length = 27 days)')\naxs[1,0].set_title('Distribution of Actions per Session (Session Length = 0 days)')\naxs[1,1].set_title('Distribution of Actions per Session (Session Length = 27 days)')\naxs[2,0].set_title('Distribution of Clicks, Carts and Orders (Session Length = 0 days)')\naxs[2,1].set_title('Distribution of Clicks, Carts and Orders (Session Length = 27 days)')\nplt.show()\n\n# axs[1].set_title('Session Length = 27 days')\n# axs[0].set_ylabel('Normalized Count')\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-05T11:09:36.920143Z","iopub.execute_input":"2022-11-05T11:09:36.920435Z","iopub.status.idle":"2022-11-05T11:09:41.909691Z","shell.execute_reply.started":"2022-11-05T11:09:36.920408Z","shell.execute_reply":"2022-11-05T11:09:41.908525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> So there is seemingly **no difference in the distributions** of <span style=\"color:olive\"> Clicks</span>, <span style=\"color:orange\">Carts</span>, <span style=\"color:red\">Orders</span> whether the session lasts 27 or 0 days <br>\n> However the distribution of <span style=\"color:blue\">Actions</span> shows, that all **sessions lasting shorter than 1 day had less than 50 actions** while for longer session the distribution looks more like a normal distribution ","metadata":{}},{"cell_type":"markdown","source":"# What are the Last actions within a Session? ","metadata":{}},{"cell_type":"code","source":"list_last = [i[-1] for i in dfs.groupby('session').agg({'type':lambda x: list(x)}).type.to_list()]\nlist_all = [item for sublist in dfs.groupby('session').agg({'type':lambda x: list(x)}).type.to_list() for item in sublist]\n","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:41.911159Z","iopub.execute_input":"2022-11-05T11:09:41.912148Z","iopub.status.idle":"2022-11-05T11:09:58.473636Z","shell.execute_reply.started":"2022-11-05T11:09:41.912109Z","shell.execute_reply":"2022-11-05T11:09:58.472582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets see how likely is it that a Session ends with an order? \n","metadata":{}},{"cell_type":"code","source":"for i in ['clicks', 'carts', 'orders']:\n    print(f'It is {np.round((list_last.count(i)*len(list_all))/(list_all.count(i)*len(list_last)), 1)} times likely that the last actions were {i}')","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:58.478470Z","iopub.execute_input":"2022-11-05T11:09:58.480761Z","iopub.status.idle":"2022-11-05T11:09:59.935237Z","shell.execute_reply.started":"2022-11-05T11:09:58.480722Z","shell.execute_reply":"2022-11-05T11:09:59.934196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> So compared to the average probabilites of clicks, carts and orders, it is half as likely that a session ends with a cart but almost twice as likely to end as an order! ","metadata":{}},{"cell_type":"markdown","source":"## Example of a session ending with an order","metadata":{}},{"cell_type":"code","source":"var = list_last.index('orders')\ndfs.query('session==@var')['type'].to_list()[-5:]","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:09:59.937068Z","iopub.execute_input":"2022-11-05T11:09:59.937741Z","iopub.status.idle":"2022-11-05T11:10:00.107806Z","shell.execute_reply.started":"2022-11-05T11:09:59.937700Z","shell.execute_reply":"2022-11-05T11:10:00.105194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Intersections of Article Ids of Different Sessions","metadata":{"execution":{"iopub.status.busy":"2022-11-02T14:16:12.787435Z","iopub.execute_input":"2022-11-02T14:16:12.787891Z","iopub.status.idle":"2022-11-02T14:16:12.792755Z","shell.execute_reply.started":"2022-11-02T14:16:12.787855Z","shell.execute_reply":"2022-11-02T14:16:12.791827Z"}}},{"cell_type":"markdown","source":"As our goal is to predict future clicked Article Ids it is interesting to look at how different sets of article ids of different sessions compare","metadata":{}},{"cell_type":"code","source":"df_aid = dfs.groupby('session').agg({'aid':lambda x: list(x), 'type':lambda x: list(x)})","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:00.109199Z","iopub.execute_input":"2022-11-05T11:10:00.109594Z","iopub.status.idle":"2022-11-05T11:10:16.041190Z","shell.execute_reply.started":"2022-11-05T11:10:00.109562Z","shell.execute_reply":"2022-11-05T11:10:16.040022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom tqdm import tqdm\n# only look at first 1000 sesssions for simplicity \ndim = 1000\ntest = df_aid.aid[:dim].to_list()\nheat_arr = np.zeros((dim, dim))\ny1  = [np.ones(dim)*i for i in range(dim)]\n\n\nfor i in tqdm(range(dim)):\n    for counter2, j in enumerate(test):\n        if test[i] != j: \n            heat_arr[i, counter2] = len(set(test[i]).intersection(set(j)))/len(set(test[i]))\n\n            \n            \n","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:16.042929Z","iopub.execute_input":"2022-11-05T11:10:16.043670Z","iopub.status.idle":"2022-11-05T11:10:25.688126Z","shell.execute_reply.started":"2022-11-05T11:10:16.043624Z","shell.execute_reply":"2022-11-05T11:10:25.686974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport plotly.graph_objs as go\nfrom itertools import cycle\n\nfig = go.Figure()\nfor i, z in enumerate(heat_arr):\n    ### to avoid clutter lets only look at a sessions which share atleast 50% of articles with at least one other session\n    if np.any(z >= 0.5):\n        fig.add_trace(\n            go.Scatter3d(\n                x = np.arange(dim),\n                y = y1[i],\n                z = z,\n                mode='lines',\n            )\n        )\nfig.update_layout(\n    scene = dict(\n       xaxis_title='Session Id',  \n       yaxis_title='Session Id', \n       zaxis_title='Overlap of Aid'),\n    width=700,\n    title=\"Overlap of Article IDs Between Different Sessions\",\n    margin=dict(r=10, l=10, b=10, t=30))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:25.689985Z","iopub.execute_input":"2022-11-05T11:10:25.690759Z","iopub.status.idle":"2022-11-05T11:10:25.952176Z","shell.execute_reply.started":"2022-11-05T11:10:25.690699Z","shell.execute_reply":"2022-11-05T11:10:25.951350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"By visualizing the overlap of only the first 1000 sessions we see that there are patterns of overlapping article Ids between sessions. Notably there are a number of some sessions which share 100% of Article Ids with one another. \n","metadata":{}},{"cell_type":"markdown","source":"# Building a simple LSTM to predict last Aid within one Session","metadata":{}},{"cell_type":"markdown","source":"Long short-term memory (LSTM) is an artificial neural network. Unlike standard feedforward neural networks, LSTM has feedback connections. Such a recurrent neural network (RNN) can process not only single data points (such as images), but also entire sequences of data. <br>\n![LSTM](https://upload.wikimedia.org/wikipedia/commons/thumb/5/56/LSTM_cell.svg/1048px-LSTM_cell.svg.png?20200923110354)","metadata":{}},{"cell_type":"markdown","source":"![](https://production-media.paperswithcode.com/methods/Screen_Shot_2020-05-25_at_8.54.27_PM.png)","metadata":{}},{"cell_type":"markdown","source":"Very nice tutorial on BI-LSTM: https://www.kaggle.com/code/ysthehurricane/next-word-prediction-bi-lstm-tutorial-easy-way","metadata":{}},{"cell_type":"markdown","source":"## <u>Objective of this first model</u>: <br>\n* Simplest LSTM model (Many-to-one) , which is able to predict the last article Id clicked within one session.\n* Not worry about overfitting for now","metadata":{"execution":{"iopub.status.busy":"2022-11-04T09:34:23.152682Z","iopub.execute_input":"2022-11-04T09:34:23.153098Z","iopub.status.idle":"2022-11-04T09:34:23.164568Z","shell.execute_reply.started":"2022-11-04T09:34:23.153063Z","shell.execute_reply":"2022-11-04T09:34:23.162611Z"}}},{"cell_type":"markdown","source":"### Starting With a Subset of the Data ","metadata":{}},{"cell_type":"code","source":"# start with only a small subset of dfs\ndfs1 = dfs.head(750000)\ndfs1.aid.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:25.953185Z","iopub.execute_input":"2022-11-05T11:10:25.953615Z","iopub.status.idle":"2022-11-05T11:10:25.989546Z","shell.execute_reply.started":"2022-11-05T11:10:25.953556Z","shell.execute_reply":"2022-11-05T11:10:25.988556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = np.arange(dfs1.aid.nunique())\n# shuffling to prevent any correlation between new labels and outcome\nnp.random.shuffle(ids)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:25.991643Z","iopub.execute_input":"2022-11-05T11:10:25.992025Z","iopub.status.idle":"2022-11-05T11:10:26.017021Z","shell.execute_reply.started":"2022-11-05T11:10:25.991985Z","shell.execute_reply":"2022-11-05T11:10:26.015965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mapping article id to integer encoding \nmap_aid = {i:j for i, j in zip(dfs1.aid.unique(), ids)}","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.018640Z","iopub.execute_input":"2022-11-05T11:10:26.019272Z","iopub.status.idle":"2022-11-05T11:10:26.097232Z","shell.execute_reply.started":"2022-11-05T11:10:26.019230Z","shell.execute_reply":"2022-11-05T11:10:26.096174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs1['aid_id'] = dfs1['aid'].map(map_aid)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.098649Z","iopub.execute_input":"2022-11-05T11:10:26.099321Z","iopub.status.idle":"2022-11-05T11:10:26.259071Z","shell.execute_reply.started":"2022-11-05T11:10:26.099279Z","shell.execute_reply":"2022-11-05T11:10:26.257950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs1.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.260712Z","iopub.execute_input":"2022-11-05T11:10:26.261367Z","iopub.status.idle":"2022-11-05T11:10:26.275962Z","shell.execute_reply.started":"2022-11-05T11:10:26.261325Z","shell.execute_reply":"2022-11-05T11:10:26.274821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Grouping the Train data by Session ","metadata":{}},{"cell_type":"code","source":"df_train = dfs1.groupby('session').agg({'aid_id':lambda x: list(x)})","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.277478Z","iopub.execute_input":"2022-11-05T11:10:26.278232Z","iopub.status.idle":"2022-11-05T11:10:26.481995Z","shell.execute_reply.started":"2022-11-05T11:10:26.278189Z","shell.execute_reply":"2022-11-05T11:10:26.480793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets for now only consider session with less than or equal 20 aids","metadata":{}},{"cell_type":"code","source":"df_train = df_train[df_train.aid_id.map(len) <= 20]\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.483820Z","iopub.execute_input":"2022-11-05T11:10:26.484216Z","iopub.status.idle":"2022-11-05T11:10:26.510383Z","shell.execute_reply.started":"2022-11-05T11:10:26.484176Z","shell.execute_reply":"2022-11-05T11:10:26.509470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Padding to make all lists of same size \nlength = max(map(len, df_train.aid_id))\nX = np.asarray([[0]*(length-len(xi)) + xi for xi in df_train.aid_id]).astype('int32')\nX","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.512689Z","iopub.execute_input":"2022-11-05T11:10:26.513387Z","iopub.status.idle":"2022-11-05T11:10:26.540278Z","shell.execute_reply.started":"2022-11-05T11:10:26.513346Z","shell.execute_reply":"2022-11-05T11:10:26.539080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train-Test-Split","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:33:54.495403Z","iopub.execute_input":"2022-11-04T11:33:54.495856Z","iopub.status.idle":"2022-11-04T11:33:54.501863Z","shell.execute_reply.started":"2022-11-04T11:33:54.495821Z","shell.execute_reply":"2022-11-04T11:33:54.500555Z"}}},{"cell_type":"markdown","source":"For now lets set `X = X[:,:-1]` (all except last aid) and `y = x[:,-1]` (last aid) for training the model. ","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_label_train, y_val_train = train_test_split(X[:,:-1], X[:, -1], test_size=0.2, shuffle=True, random_state=42)\ny_train = tf.keras.utils.to_categorical(y_label_train, num_classes=dfs1.aid.nunique())\n# onehot encoding on labels corresponding to num_classes\ny_val = tf.keras.utils.to_categorical(y_val_train, num_classes=dfs1.aid.nunique())\nX_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:26.541898Z","iopub.execute_input":"2022-11-05T11:10:26.542232Z","iopub.status.idle":"2022-11-05T11:10:27.301651Z","shell.execute_reply.started":"2022-11-05T11:10:26.542197Z","shell.execute_reply":"2022-11-05T11:10:27.300456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train = np.expand_dims(X_train, axis=2)\n# X_val = np.expand_dims(X_val, axis=2)\n# X_train.shape, y_label_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:27.303284Z","iopub.execute_input":"2022-11-05T11:10:27.304815Z","iopub.status.idle":"2022-11-05T11:10:27.309729Z","shell.execute_reply.started":"2022-11-05T11:10:27.304771Z","shell.execute_reply":"2022-11-05T11:10:27.308363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-11-05T11:10:27.311585Z","iopub.execute_input":"2022-11-05T11:10:27.312047Z","iopub.status.idle":"2022-11-05T11:10:28.111690Z","shell.execute_reply.started":"2022-11-05T11:10:27.312006Z","shell.execute_reply":"2022-11-05T11:10:28.110468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### BASIC Bidirectional-LSTM Model Architecture","metadata":{}},{"cell_type":"code","source":"model = tf.keras.Sequential()\n# Adding an Embbeding Layer\nmodel.add(layers.Embedding(dfs1.aid.nunique(), 20, input_length=X_train.shape[1]))\nmodel.add(layers.Bidirectional(layers.LSTM(64, dropout=0.2)))\nmodel.add(layers.Dense(dfs1.aid.nunique(), activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:28.119024Z","iopub.execute_input":"2022-11-05T11:10:28.119327Z","iopub.status.idle":"2022-11-05T11:10:33.077868Z","shell.execute_reply.started":"2022-11-05T11:10:28.119299Z","shell.execute_reply":"2022-11-05T11:10:33.076790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:33.079579Z","iopub.execute_input":"2022-11-05T11:10:33.079994Z","iopub.status.idle":"2022-11-05T11:10:33.087297Z","shell.execute_reply.started":"2022-11-05T11:10:33.079951Z","shell.execute_reply":"2022-11-05T11:10:33.086048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import RMSprop, Adam\n\noptimizer = Adam(learning_rate=0.01)\nmodel.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:10:33.089140Z","iopub.execute_input":"2022-11-05T11:10:33.089883Z","iopub.status.idle":"2022-11-05T11:10:33.106634Z","shell.execute_reply.started":"2022-11-05T11:10:33.089847Z","shell.execute_reply":"2022-11-05T11:10:33.105702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=3, validation_data=(X_val, y_val), verbose=2, batch_size=64)\nmodel.save('./model1.h5')\n# # Loadin pre trained model from cache\n# recon_model = keras.models.load_model(\"model1.h5\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-05T11:10:33.108013Z","iopub.execute_input":"2022-11-05T11:10:33.108597Z","iopub.status.idle":"2022-11-05T11:11:26.159053Z","shell.execute_reply.started":"2022-11-05T11:10:33.108555Z","shell.execute_reply":"2022-11-05T11:11:26.157607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\nax1.plot(history.history['accuracy'])\nax2.plot(history.history['loss'], color='Orange')\n# plt.plot(history.history['val_acc'])\n# plt.title('model accuracy')\n# plt.ylabel('accuracy')\nax2.set_xlabel('epoch')\nax1.set_ylabel('accuracy')\nax2.set_ylabel('loss')\nax1.set_title('Train Accuracy/Epoch')\nax2.set_title('Train Loss/Epoch')\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:11:26.165870Z","iopub.execute_input":"2022-11-05T11:11:26.166636Z","iopub.status.idle":"2022-11-05T11:11:26.517552Z","shell.execute_reply.started":"2022-11-05T11:11:26.166594Z","shell.execute_reply":"2022-11-05T11:11:26.516441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, acc = model.evaluate(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T11:11:26.519091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some tests","metadata":{"execution":{"iopub.status.busy":"2022-11-03T21:30:26.261283Z","iopub.execute_input":"2022-11-03T21:30:26.262340Z","iopub.status.idle":"2022-11-03T21:30:26.267658Z","shell.execute_reply.started":"2022-11-03T21:30:26.262286Z","shell.execute_reply":"2022-11-03T21:30:26.266455Z"}}},{"cell_type":"code","source":"Next_aid = model.predict(X_train, verbose=0)\nfig, axs = plt.subplots(2, 2, figsize=(15,11))\nnp.random.seed(42)\nfor i in range(2):\n    for j in range(2):\n        val = np.random.randint(X_train.shape[0]) \n        max_val = np.where(Next_aid[val] == np.max(Next_aid[val]))\n        axs[i,j].plot(Next_aid[val], label='softmax output')\n        axs[i,j].plot([y_label_train[val], y_label_train[val]], [1, 0], alpha=0.5, c='red', label='true label')\n        axs[i,j].legend()\n        axs[i,j].set_title(f'Train Session {val}')\n        axs[i,j].annotate(xy=(y_label_train[val],0.1), text=f'Predicted Label:{max_val[0][0]} \\n True Label:{y_label_train[val]}', color='red')\n        axs[i,j].set_xlabel('Article ID')\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'So for now we have a model which has a accuracy of {np.round(acc,3)} on the train data, but is clearly overfitting!')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Next steps:\n* Increase Input sample\n* Change Loss function to better fit the competition\n* Add dropout\n* Number of features (article Ids)?\n* Model Validation\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-04T09:52:00.356093Z","iopub.execute_input":"2022-11-04T09:52:00.356515Z","iopub.status.idle":"2022-11-04T09:52:00.363456Z","shell.execute_reply.started":"2022-11-04T09:52:00.356479Z","shell.execute_reply":"2022-11-04T09:52:00.362142Z"}}},{"cell_type":"markdown","source":"## WIP :)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T12:13:51.385894Z","iopub.execute_input":"2022-11-04T12:13:51.386280Z","iopub.status.idle":"2022-11-04T12:13:51.391530Z","shell.execute_reply.started":"2022-11-04T12:13:51.386247Z","shell.execute_reply":"2022-11-04T12:13:51.390403Z"}}}]}