{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, we will display time series EDA for products which have the most clicks/carts/orders. <br>\nWe will use RAPIDS cuDF to process dataframes and seaborn to display EDA. <br>\n\nMany products doesn't exhibit regular pattern.<br>\nA product with a lot of clicks/carts/orders one week, will not have necessary a lot of clicks/carts/orders next week. <br>\n\nLet's have a look to AID n°485256 below : almost 97 000 sessions with a click on this product day 235, an no click all other days...<br>\n\nSo there are many outliers : we should build models and make predictions only for products which have some regular patterns, otherwise we will overfit. <br>\nDo you agree ?<br>\n\nThere is a Kaggle discussion about those strange product time series [here](https://www.kaggle.com/competitions/otto-recommender-system/discussion/368373).<br>","metadata":{}},{"cell_type":"markdown","source":"# Libraries & Train data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('max_columns', 100)\npd.set_option('max_rows', 200)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cudf, cupy\nprint('Using RAPIDS version',cudf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-24T22:05:35.296007Z","iopub.execute_input":"2022-11-24T22:05:35.296728Z","iopub.status.idle":"2022-11-24T22:05:38.124970Z","shell.execute_reply.started":"2022-11-24T22:05:35.296688Z","shell.execute_reply":"2022-11-24T22:05:38.123830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntrain.ts = cudf.to_datetime((train.ts + 2*60*60) * 1e9)\nprint('Train min date and max date are:', train.ts.min(),'and', train.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:05:38.126958Z","iopub.execute_input":"2022-11-24T22:05:38.127277Z","iopub.status.idle":"2022-11-24T22:05:59.795353Z","shell.execute_reply.started":"2022-11-24T22:05:38.127247Z","shell.execute_reply":"2022-11-24T22:05:59.793522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"dayofyear\"] = (train.ts.dt.dayofyear).astype(np.int16)\ntrain[\"weekofyear\"] = ((train[\"dayofyear\"]-3)//7).astype(np.int8)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:05:59.796607Z","iopub.execute_input":"2022-11-24T22:05:59.796953Z","iopub.status.idle":"2022-11-24T22:05:59.909749Z","shell.execute_reply.started":"2022-11-24T22:05:59.796923Z","shell.execute_reply":"2022-11-24T22:05:59.908636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"weekofyear\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:05:59.912690Z","iopub.execute_input":"2022-11-24T22:05:59.913044Z","iopub.status.idle":"2022-11-24T22:06:00.092504Z","shell.execute_reply.started":"2022-11-24T22:05:59.913008Z","shell.execute_reply":"2022-11-24T22:06:00.091457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Session with AID by week, day and type\n\nWe want to know in how many sessions, an AID is, so I'm using .count()","metadata":{}},{"cell_type":"code","source":"aid_per_week = train.groupby([\"aid\", \"weekofyear\", \"type\"])[\"session\"].count().reset_index()\naid_per_day = train.groupby([\"aid\", \"dayofyear\", 'type'])[\"session\"].count().reset_index()\ndisplay(aid_per_week.head(10))\ndisplay(aid_per_day.head(10))","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:00.094267Z","iopub.execute_input":"2022-11-24T22:06:00.094623Z","iopub.status.idle":"2022-11-24T22:06:00.893949Z","shell.execute_reply.started":"2022-11-24T22:06:00.094586Z","shell.execute_reply":"2022-11-24T22:06:00.893058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the first AID : AID 592716 during week 30 with 0(clicks) appears 4 times. Great !\ntrain.loc[train[\"aid\"]==592716]","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:00.895286Z","iopub.execute_input":"2022-11-24T22:06:00.895750Z","iopub.status.idle":"2022-11-24T22:06:01.000840Z","shell.execute_reply.started":"2022-11-24T22:06:00.895712Z","shell.execute_reply":"2022-11-24T22:06:00.999778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The 20 most important AIDS by week and by type","metadata":{}},{"cell_type":"code","source":"most_aid_per_week = {}\nfor w in aid_per_week[\"weekofyear\"].unique().values.tolist():\n    most_aid_per_week[w]={}\n    for t in [0, 1, 2]: # 0 : click / 1 : Cart / 2 : Order\n        most_aid_per_week[w][t] = aid_per_week.loc[(aid_per_week[\"type\"] == t)&(aid_per_week[\"weekofyear\"]==w)].sort_values(\"session\", ascending=False)[:20][\"aid\"].values.tolist()\n    \nprint(most_aid_per_week)","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:01.002531Z","iopub.execute_input":"2022-11-24T22:06:01.002902Z","iopub.status.idle":"2022-11-24T22:06:01.161054Z","shell.execute_reply.started":"2022-11-24T22:06:01.002866Z","shell.execute_reply":"2022-11-24T22:06:01.159997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types_ = {0:\"clicks\", 1:\"carts\", 2:\"orders\"}","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:01.162586Z","iopub.execute_input":"2022-11-24T22:06:01.163057Z","iopub.status.idle":"2022-11-24T22:06:01.169059Z","shell.execute_reply.started":"2022-11-24T22:06:01.163009Z","shell.execute_reply":"2022-11-24T22:06:01.167911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most important AID during week 30\n## The first 10 AID with the most CLICKS","metadata":{}},{"cell_type":"code","source":"colors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nweek = 30 # \ntype = 0  # 0 : click / 1 : Cart / 2 : Order\nn=10\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.2)#, left=.3)\nax = ax.flatten()\ng=0\nfor aid in most_aid_per_week[week][type][:10]:\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"{aid} - {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.95)\nfig.suptitle(f\"The {n} aids with the most {types_[type]} during week {week} per day \");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:10:33.593643Z","iopub.execute_input":"2022-11-24T22:10:33.594033Z","iopub.status.idle":"2022-11-24T22:10:39.066068Z","shell.execute_reply.started":"2022-11-24T22:10:33.594000Z","shell.execute_reply":"2022-11-24T22:10:39.065100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1502122 : many clicks, but no order<br>\n1083665, 610733 : many clicks, carts, orders just one day","metadata":{}},{"cell_type":"markdown","source":"## The first 10 AID with the most CARTS","metadata":{}},{"cell_type":"code","source":"colors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nweek = 30 # \ntype = 1  # 0 : click / 1 : Cart / 2 : Order\nn=10\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.2)#, left=.3)\nax = ax.flatten()\ng=0\nfor aid in most_aid_per_week[week][type][:10]:\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"{aid} - {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.95)\nfig.suptitle(f\"The {n} aids with the most {types_[type]} during week {week} per day \");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:09:47.136161Z","iopub.execute_input":"2022-11-24T22:09:47.137253Z","iopub.status.idle":"2022-11-24T22:09:52.378126Z","shell.execute_reply.started":"2022-11-24T22:09:47.137203Z","shell.execute_reply":"2022-11-24T22:09:52.377198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AID n°152547 had only 2 days with almost 10 000 carts, and none the other days.\n## The first 10 AID with the most ORDERS","metadata":{}},{"cell_type":"code","source":"colors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nweek = 30\ntype = 2\nn=10\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.1)#, left=.3)\nax = ax.flatten()\ng=0\nfor aid in most_aid_per_week[week][type][:10]:\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"{aid} - {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.95)\nfig.suptitle(f\"The {n} aids with the most {types_[type]} during week {week} per day \");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:10.820858Z","iopub.execute_input":"2022-11-24T22:06:10.821316Z","iopub.status.idle":"2022-11-24T22:06:15.986914Z","shell.execute_reply.started":"2022-11-24T22:06:10.821270Z","shell.execute_reply":"2022-11-24T22:06:15.985950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The AID with the most CLICKS each week","metadata":{}},{"cell_type":"code","source":"type=0\nlp = [most_aid_per_week[30][type][0], most_aid_per_week[31][type][0], most_aid_per_week[32][type][0], most_aid_per_week[33][type][0]]\nn = len(lp)\n\ncolors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.1)#, left=.3)\nax = ax.flatten()\ng=0\nfor w, aid in enumerate(lp):\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"Week {w+30} : {aid} had the most {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.90)\nfig.suptitle(f\"The aid with the most {types_[type]} for each week\");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:35.812111Z","iopub.execute_input":"2022-11-24T22:06:35.813071Z","iopub.status.idle":"2022-11-24T22:06:37.789084Z","shell.execute_reply.started":"2022-11-24T22:06:35.813002Z","shell.execute_reply":"2022-11-24T22:06:37.788099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The AID with the most CARTS each week","metadata":{}},{"cell_type":"code","source":"type = 1\nlp = [most_aid_per_week[30][type][0], most_aid_per_week[31][type][0], most_aid_per_week[32][type][0], most_aid_per_week[33][type][0]]\nn = len(lp)\n\ncolors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.1)#, left=.3)\nax = ax.flatten()\ng=0\nfor w, aid in enumerate(lp):\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"Week {w+30} : {aid} had the most {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.90)\nfig.suptitle(f\"The aid with the most {types_[type]} for each week\");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:41.847803Z","iopub.execute_input":"2022-11-24T22:06:41.848196Z","iopub.status.idle":"2022-11-24T22:06:43.753993Z","shell.execute_reply.started":"2022-11-24T22:06:41.848160Z","shell.execute_reply":"2022-11-24T22:06:43.753064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"485256 has CARTS only one day","metadata":{}},{"cell_type":"markdown","source":"# The AID with the most ORDERS each week","metadata":{}},{"cell_type":"code","source":"type = 2\nlp = [most_aid_per_week[30][type][0], most_aid_per_week[31][type][0], most_aid_per_week[32][type][0], most_aid_per_week[33][type][0]]\nn = len(lp)\n\ncolors = np.array( [(0,0,1),(1,0.5,0),(1,0,0)] )\n\nfig, ax = plt.subplots(nrows = n, ncols = 3, figsize = (20, 3*n), sharex=True)\nplt.subplots_adjust(hspace = 0.2, wspace = 0.1)#, left=.3)\nax = ax.flatten()\ng=0\nfor w, aid in enumerate(lp):\n    for t in [0, 1, 2]:\n        sns.scatterplot(\n            x=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"dayofyear\"].to_array(), \n            y=aid_per_day.loc[(aid_per_day[\"aid\"]==aid)&(aid_per_day[\"type\"]==t)][\"session\"].to_array(), \n            c=colors[t],\n            ax=ax[g]\n        )\n        ax[g].set_title(f\"Week {w+30} : {aid} had the most {types_[t]}\")\n        ax[g].set_xlabel(\"days\")\n        ax[g].set_ylabel(\"Sessions\")\n        g+=1\n\nfig.subplots_adjust(top=0.90)\nfig.suptitle(f\"The aid with the most {types_[type]} for each week\");","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:43.831888Z","iopub.execute_input":"2022-11-24T22:06:43.832227Z","iopub.status.idle":"2022-11-24T22:06:46.129152Z","shell.execute_reply.started":"2022-11-24T22:06:43.832189Z","shell.execute_reply":"2022-11-24T22:06:46.128171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Were there a sales promotion on day 237 for AID 876493 ?","metadata":{}},{"cell_type":"code","source":"aid_per_day.loc[(aid_per_day[\"aid\"]==876493)&(aid_per_day[\"type\"]==2)]","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:49.404214Z","iopub.execute_input":"2022-11-24T22:06:49.405338Z","iopub.status.idle":"2022-11-24T22:06:49.435945Z","shell.execute_reply.started":"2022-11-24T22:06:49.405292Z","shell.execute_reply":"2022-11-24T22:06:49.434889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aid_per_day.loc[(aid_per_day[\"aid\"]==876493)&(aid_per_day[\"type\"]==0)]","metadata":{"execution":{"iopub.status.busy":"2022-11-24T22:06:50.201269Z","iopub.execute_input":"2022-11-24T22:06:50.201654Z","iopub.status.idle":"2022-11-24T22:06:50.233071Z","shell.execute_reply.started":"2022-11-24T22:06:50.201610Z","shell.execute_reply":"2022-11-24T22:06:50.232043Z"},"trusted":true},"execution_count":null,"outputs":[]}]}