{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.express as px\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom IPython.core.display import display, HTML\nimport ipywidgets as widgets\nfrom IPython.display import display,clear_output\nfrom ipywidgets import Output\nfrom ipywidgets import TwoByTwoLayout\n# Utils widgets\nfrom ipywidgets import Button, Layout, jslink, IntText, IntSlider, Box, VBox\n\nfrom scipy.stats import pearsonr\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport plotly.io as pio\npio.templates\nfrom PIL import Image\nfrom IPython.display import Image as img\nfrom plotly.offline import plot, iplot, init_notebook_mode\nimport random\npd.options.mode.chained_assignment = None  # default='warn'\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# Load dataset \narticle = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomer = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\ntransaction = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:36:08.957090Z","iopub.execute_input":"2022-02-10T09:36:08.957482Z","iopub.status.idle":"2022-02-10T09:37:31.598420Z","shell.execute_reply.started":"2022-02-10T09:36:08.957377Z","shell.execute_reply":"2022-02-10T09:37:31.597261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Content\n\nThis notebook is built to explore the dataset.\n\nIt is an ongoing work, and please stay tuned.\n\n* Null values of each dataset\n* Article ranking based on different variables\n* Histgram of customer status data broken down by age\n* Purchase metrics change over the time (number of customers/total volume/number of articles purchased per day)\n* How long does it take a customer to make the next purchase, on average?\n* Average age of customer changes over the time\n* Top 12 articles sold in the week before training dataset ending\n* number of articles purchased by customer\n* How many (much %) of users doesn't purchase equal or more than 12 articles?\n* Number of daily transactions per sales channel\n","metadata":{}},{"cell_type":"markdown","source":"# Null values of each dataset","metadata":{}},{"cell_type":"code","source":"# article.csv\narticle.isna().sum()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-02-10T09:37:31.600034Z","iopub.execute_input":"2022-02-10T09:37:31.600285Z","iopub.status.idle":"2022-02-10T09:37:31.773748Z","shell.execute_reply.started":"2022-02-10T09:37:31.600256Z","shell.execute_reply":"2022-02-10T09:37:31.772810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# customer.csv\ncustomer.isna().sum()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-02-10T09:37:31.775229Z","iopub.execute_input":"2022-02-10T09:37:31.776095Z","iopub.status.idle":"2022-02-10T09:37:32.412568Z","shell.execute_reply.started":"2022-02-10T09:37:31.776046Z","shell.execute_reply":"2022-02-10T09:37:32.411851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transaction_train.csv\ntransaction.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-10T09:37:32.415295Z","iopub.execute_input":"2022-02-10T09:37:32.415943Z","iopub.status.idle":"2022-02-10T09:37:39.312735Z","shell.execute_reply.started":"2022-02-10T09:37:32.415893Z","shell.execute_reply":"2022-02-10T09:37:39.311853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Article ranking based on different variables\n1. Choose a variable you are interested in (e.g. prod_name)\n2. Choose number of rows of the result (e.g. 20 rows)\n3. Click the button and you will see the ranking regarding how many articles per variable value\n\nAs an example, let's see the top 20 product names in terms of number of articles.","metadata":{}},{"cell_type":"code","source":"def number_article(v,row):\n    result = article.groupby([v]).nunique().reset_index()[[v,'article_id']].sort_values(by='article_id',ascending = False).rename(columns={\"article_id\": \"nr_article\"})\n    return result.head(row)\nnumber_article('prod_name',20)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:37:39.314204Z","iopub.execute_input":"2022-02-10T09:37:39.314505Z","iopub.status.idle":"2022-02-10T09:37:40.204467Z","shell.execute_reply.started":"2022-02-10T09:37:39.314462Z","shell.execute_reply":"2022-02-10T09:37:40.203610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### You may need to fork this notebook to interact with the widget from your own end.","metadata":{}},{"cell_type":"code","source":"article_column = list(article.columns)\narticle_column.remove('article_id')\n\n# display widgets\noutput = Output()\nstart = Button(description=\"Click me\")\nstart.style.button_color = 'lightblue'\n\nv_widget = widgets.Dropdown(\n    options=article_column,\n    value=list(list(article_column))[1],\n    description='variable',\n    disabled=False,\n)\n\nrow_widgets = widgets.IntSlider(\n    value=20,\n    min=0,\n    max=100,\n    step=5,\n    description='row',\n    disabled=False,\n    continuous_update=False,\n    orientation='horizontal',\n    readout=True,\n    readout_format='d',\n)\n\n\ndef click_start(b):\n    with output:\n        clear_output()\n        print(number_article(v_widget.value,\n                            row_widgets.value))\n        \n\nstart.on_click(click_start)\n\ndisplay(v_widget,\n        row_widgets,\n        start,\n        output)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:37:40.205822Z","iopub.execute_input":"2022-02-10T09:37:40.206094Z","iopub.status.idle":"2022-02-10T09:37:40.249912Z","shell.execute_reply.started":"2022-02-10T09:37:40.206060Z","shell.execute_reply":"2022-02-10T09:37:40.249321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Histgram of customer status data broken down by age","metadata":{}},{"cell_type":"code","source":"# replace club_member_status null values with 'None'\n# replace Active null values with 0\n# replace FN null values with 0\n\ncustomer['club_member_status'] = customer['club_member_status'].fillna('None')\ncustomer['Active'] = customer['Active'].fillna(0)\ncustomer['FN'] = customer['FN'].fillna(0)\ncustomer['fashion_news_frequency'] = customer['fashion_news_frequency'].fillna('nan')\n\nfig = px.histogram(customer, x=\"age\", color=\"club_member_status\")\nfig.update_layout(\n    title_text='Histgram of customer age per club_member_status', # title of plot\n    xaxis_title_text='Age', # xaxis label\n    yaxis_title_text='Count', # yaxis label\n    bargap=0.2, # gap between bars of adjacent location coordinates\n    bargroupgap=0.1 # gap between bars of the same location coordinates\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:37:40.251046Z","iopub.execute_input":"2022-02-10T09:37:40.251326Z","iopub.status.idle":"2022-02-10T09:37:48.286030Z","shell.execute_reply.started":"2022-02-10T09:37:40.251292Z","shell.execute_reply":"2022-02-10T09:37:48.284856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(customer, x=\"age\", color=\"Active\")\nfig.update_layout(\n    title_text='Histgram of customer age per Active status', # title of plot\n    xaxis_title_text='Age', # xaxis label\n    yaxis_title_text='Count', # yaxis label\n    bargap=0.2, # gap between bars of adjacent location coordinates\n    bargroupgap=0.1 # gap between bars of the same location coordinates\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:37:48.288981Z","iopub.execute_input":"2022-02-10T09:37:48.289785Z","iopub.status.idle":"2022-02-10T09:37:54.982768Z","shell.execute_reply.started":"2022-02-10T09:37:48.289741Z","shell.execute_reply":"2022-02-10T09:37:54.981204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(customer, x=\"age\", color=\"FN\")\nfig.update_layout(\n    title_text='Histgram of customer age per FN status', # title of plot\n    xaxis_title_text='Age', # xaxis label\n    yaxis_title_text='Count', # yaxis label\n    bargap=0.2, # gap between bars of adjacent location coordinates\n    bargroupgap=0.1 # gap between bars of the same location coordinates\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:37:54.985798Z","iopub.execute_input":"2022-02-10T09:37:54.986092Z","iopub.status.idle":"2022-02-10T09:38:02.649595Z","shell.execute_reply.started":"2022-02-10T09:37:54.986061Z","shell.execute_reply":"2022-02-10T09:38:02.648704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(customer, x=\"age\", color=\"fashion_news_frequency\")\nfig.update_layout(\n    title_text='Histgram of customer age per fashion_news_frequency', # title of plot\n    xaxis_title_text='Age', # xaxis label\n    yaxis_title_text='Count', # yaxis label\n    bargap=0.2, # gap between bars of adjacent location coordinates\n    bargroupgap=0.1 # gap between bars of the same location coordinates\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:38:02.653021Z","iopub.execute_input":"2022-02-10T09:38:02.653595Z","iopub.status.idle":"2022-02-10T09:38:09.123102Z","shell.execute_reply.started":"2022-02-10T09:38:02.653537Z","shell.execute_reply":"2022-02-10T09:38:09.119557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Purchase metrics change over the time","metadata":{}},{"cell_type":"code","source":"# aggregate the transaction by date\ntransaction_aggr = transaction.groupby(['t_dat']).nunique().reset_index()[['t_dat','customer_id','article_id','sales_channel_id']]\n\n# create a column showing sum per user per day\ntransaction_aggr['sum'] = transaction.groupby(['t_dat']).sum().reset_index()[['price']]\ntransaction_aggr = transaction_aggr.rename(columns={\"customer_id\": \"nr_customer\",\n                                                   \"article_id\":\"nr_article\",\n                                                   \"sales_channel_id\":\"nr_sales_channels\",\n                                                   \"sum\":\"total_volume\"})\n# plot the chart\nfig = go.Figure()\n\nvariables = ['nr_customer','nr_article','nr_sales_channels','total_volume']\n\nfor v in variables:\n    fig.add_trace(go.Scatter(mode=\"lines\", x=transaction_aggr[\"t_dat\"], y=transaction_aggr[v], name=v))\n\nfig.update_xaxes(\n    rangeslider_visible=True,\n    rangeselector=dict(\n        buttons=list([\n            dict(count=1, label=\"1m\", step=\"month\", stepmode=\"backward\"),\n            dict(count=6, label=\"6m\", step=\"month\", stepmode=\"backward\"),\n            dict(count=1, label=\"YTD\", step=\"year\", stepmode=\"todate\"),\n            dict(count=1, label=\"1y\", step=\"year\", stepmode=\"backward\"),\n            dict(step=\"all\")\n        ])\n    )\n)\n\nfig.update_layout(\n    title_text='Purchase metrics change over the time' # title of plot\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:38:09.125849Z","iopub.execute_input":"2022-02-10T09:38:09.126536Z","iopub.status.idle":"2022-02-10T09:38:57.984583Z","shell.execute_reply.started":"2022-02-10T09:38:09.126480Z","shell.execute_reply":"2022-02-10T09:38:57.983154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How long does it take a customer to make the next purchase, on average?\n\nAs the time goes, it takes users shorter time to make the next purchase. This trend appears especially since 2020 April, and I guess it is closely related with the pandamic when people are more likely to purchase online.","metadata":{}},{"cell_type":"code","source":"# select relevant columns\ntransaction_date = transaction[['t_dat','customer_id']]\n\n# create a column showing the time the next order was made, grouping by customer id\ntransaction_date['shift_t_dat'] = transaction_date.groupby(\"customer_id\").shift(-1)\n\n# remove rows that have null\ntransaction_date = transaction_date.dropna()\n\n# calculate the difference between last order and the next one\ntransaction_date['t_dat_dff'] = pd.to_datetime(transaction_date['shift_t_dat']) - pd.to_datetime(transaction_date['t_dat'])\n\n# remove time difference equaling to 0, which indicates users making the orders in the same day\ntransaction_date_remove_zero = transaction_date.loc[transaction_date['t_dat_dff'] != '0 day']\n\n# take the average number of days grouping by date\ntransaction_date_remove_zero_aggr = transaction_date_remove_zero.groupby(['t_dat']).mean().reset_index()\ntransaction_date_remove_zero_aggr['t_dat_dff'] = pd.to_timedelta(transaction_date_remove_zero_aggr.t_dat_dff, errors='coerce').dt.days\n\n# plot the chart\nfig = go.Figure()\n\nfig.add_trace(go.Scatter(mode=\"lines\", x=transaction_date_remove_zero_aggr[\"t_dat\"], y=transaction_date_remove_zero_aggr['t_dat_dff'], name=\"Number of days since last purchase\"))\nfig.update_layout(\n    title_text='Average number of days the next order is purchased on the same customer' # title of plot\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:38:57.985890Z","iopub.execute_input":"2022-02-10T09:38:57.986907Z","iopub.status.idle":"2022-02-10T09:39:46.113435Z","shell.execute_reply.started":"2022-02-10T09:38:57.986853Z","shell.execute_reply":"2022-02-10T09:39:46.112552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Average age of customer changes over the time\n\nMore younger customers come to buy as the time goes.","metadata":{}},{"cell_type":"code","source":"# join customer dataset with transactions\ncustomer_transaction = customer.merge(transaction,how='left', on=None, left_on='customer_id', right_on='customer_id', suffixes=('_x', '_y'))\n# age of each customer\ncustomer_transaction_aggr = customer_transaction.groupby(['t_dat','customer_id']).mean().reset_index()[['age','customer_id','t_dat']]\n# mean age of customer on each day\nage_transaction_aggr = customer_transaction_aggr.groupby(['t_dat']).mean().reset_index()[['age','t_dat']]\nage_transaction_aggr['t_dat'] = pd.to_datetime(age_transaction_aggr['t_dat'])\n\n# plot the chart\nfig = px.scatter(age_transaction_aggr,x=\"t_dat\", y=\"age\", trendline=\"ols\",title=\"Average age of customer changes along the time\")\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:39:46.114786Z","iopub.execute_input":"2022-02-10T09:39:46.115045Z","iopub.status.idle":"2022-02-10T09:40:50.054660Z","shell.execute_reply.started":"2022-02-10T09:39:46.115013Z","shell.execute_reply":"2022-02-10T09:40:50.053639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Top 12 articles sold in the week before training dataset ending\n\nFor time series prediction, usually we will see the most recent observations have most influence on the predicted outcomes.\n\nThus, let's take a look at the top 12 articles in the last week of training dataset.","metadata":{}},{"cell_type":"code","source":"# select data in the last week\ntransaction_last_week = transaction.loc[transaction['t_dat'].isin(['2020-09-22',\n                                                                   '2020-09-21',\n                                                                   '2020-09-20',\n                                                                   '2020-09-19',\n                                                                   '2020-09-18',\n                                                                   '2020-09-17',\n                                                                   '2020-09-16'])]\n# get the top 12 articles sold in the last week\ntop_12_last_week = transaction_last_week.groupby(['article_id']).count().reset_index().sort_values(by='customer_id',ascending = False).head(12)\n# to get detail of these 12 articles\ntop_12_last_week_info = top_12_last_week.merge(article,how='inner', on=None, left_on='article_id', right_on='article_id', suffixes=('_x', '_y'))\n# Select interested columns\ntop_12_last_week_info[['article_id','prod_name','product_type_name','product_group_name','colour_group_name','perceived_colour_value_name','department_name','index_name']]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:40:50.056000Z","iopub.execute_input":"2022-02-10T09:40:50.056277Z","iopub.status.idle":"2022-02-10T09:40:51.932428Z","shell.execute_reply.started":"2022-02-10T09:40:50.056243Z","shell.execute_reply":"2022-02-10T09:40:51.931505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# number of articles purchased by customer","metadata":{}},{"cell_type":"code","source":"transaction_customer_aggr = transaction.groupby(['customer_id']).nunique().reset_index()\nprint(\"Number of articles purchased per customer - mean \" + str(round(transaction_customer_aggr['article_id'].mean(),1)))\nprint(\"Number of articles purchased per customer - median \" + str(round(transaction_customer_aggr['article_id'].median(),1)))\nprint(\"Number of articles purchased per customer - 75th percentile \" + str(round(transaction_customer_aggr['article_id'].quantile(.75),1)))\nprint(\"Number of articles purchased per customer - 95th percentile \" + str(round(transaction_customer_aggr['article_id'].quantile(.95),1)))\nprint(\"Number of articles purchased per customer - 99th percentile \" + str(round(transaction_customer_aggr['article_id'].quantile(.99),1)))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:40:51.933638Z","iopub.execute_input":"2022-02-10T09:40:51.933886Z","iopub.status.idle":"2022-02-10T09:41:46.002028Z","shell.execute_reply.started":"2022-02-10T09:40:51.933853Z","shell.execute_reply":"2022-02-10T09:41:46.001118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many (much %) of users doesn't purchase equal or more than 12 articles?","metadata":{}},{"cell_type":"code","source":"print(str(transaction_customer_aggr['customer_id'].nunique()) + \" customers in total\")\n\nprint(\"and \" + str(transaction_customer_aggr.loc[transaction_customer_aggr['article_id'] < 12]['customer_id'].nunique()) + \" customers purchased articles fewer than 12.\")\n\nprint(str(round(transaction_customer_aggr.loc[transaction_customer_aggr['article_id'] < 12]['customer_id'].nunique() * 100 /\n     transaction_customer_aggr['customer_id'].nunique(),1)) + \" percentage of customers purchased fewer than 12 articles.\")","metadata":{"execution":{"iopub.status.busy":"2022-02-10T09:41:46.003726Z","iopub.execute_input":"2022-02-10T09:41:46.004371Z","iopub.status.idle":"2022-02-10T09:41:50.280478Z","shell.execute_reply.started":"2022-02-10T09:41:46.004325Z","shell.execute_reply":"2022-02-10T09:41:50.277303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# number of daily transactions per sales channel","metadata":{}},{"cell_type":"code","source":"transaction_sales = transaction.groupby(['t_dat','sales_channel_id']).nunique().reset_index()\nfig = px.line(transaction_sales, x='t_dat', y='customer_id', color='sales_channel_id',title=\"Nr of articles purchased per sales channel\")\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-10T09:50:32.403326Z","iopub.execute_input":"2022-02-10T09:50:32.403872Z","iopub.status.idle":"2022-02-10T09:51:17.560482Z","shell.execute_reply.started":"2022-02-10T09:50:32.403835Z","shell.execute_reply":"2022-02-10T09:51:17.559460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}