{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Quoting and Conclusion\n\n@astrung published a discussion and a notebook about user's distribution. \\\nDiscussion: https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/312653 \\\nNotebook: https://www.kaggle.com/astrung/eda-extract-user-metadata-to-apply-deep-model/notebook \\\nPlease UPVOTE them too!\n\nThanks for your great interesting Notebooks and Notebooks!\n\nI validated its strategy by hold-out method. \\\nValid term is from 2020-09-16 to 2020-09-22 (both included). \\\nI shift the transactions' date 1 week later in training data for strict validation. (We cut the last week's transactions, so the amount of transactions of September get lowers.)\n\nThe insight abought this validation is in this discussion. \\\nhttps://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/315587","metadata":{}},{"cell_type":"code","source":"from datetime import timedelta\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.061427,"end_time":"2022-03-13T09:33:46.296967","exception":false,"start_time":"2022-03-13T09:33:46.23554","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:19:35.044935Z","iopub.execute_input":"2022-03-29T01:19:35.045387Z","iopub.status.idle":"2022-03-29T01:19:35.049588Z","shell.execute_reply.started":"2022-03-29T01:19:35.045355Z","shell.execute_reply":"2022-03-29T01:19:35.048772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ndf['t_dat'] = pd.to_datetime(df['t_dat'], format=\"%Y-%m-%d\")\ndf = df[df['t_dat'] <= pd.to_datetime('2020-09-15')]\ndf.head()","metadata":{"papermill":{"duration":66.794003,"end_time":"2022-03-13T09:34:53.224756","exception":false,"start_time":"2022-03-13T09:33:46.430753","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:21:07.102966Z","iopub.execute_input":"2022-03-29T01:21:07.104336Z","iopub.status.idle":"2022-03-29T01:21:54.840505Z","shell.execute_reply.started":"2022-03-29T01:21:07.104275Z","shell.execute_reply":"2022-03-29T01:21:54.839539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['t_dat'] = pd.to_datetime(df['t_dat'], format=\"%Y-%m-%d\")\n\n# Shift transactions for strict validation!\ndf['t_dat'] = df['t_dat'] + timedelta(weeks=1)\n\ndf['month'] = df['t_dat'].dt.strftime('%m')\ndf['year'] = df['t_dat'].dt.strftime('%Y')\ndf.head()","metadata":{"papermill":{"duration":494.575035,"end_time":"2022-03-13T09:43:07.842859","exception":false,"start_time":"2022-03-13T09:34:53.267824","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:22:29.779672Z","iopub.execute_input":"2022-03-29T01:22:29.780157Z","iopub.status.idle":"2022-03-29T01:30:08.014333Z","shell.execute_reply.started":"2022-03-29T01:22:29.780123Z","shell.execute_reply":"2022-03-29T01:30:08.01333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df['year'] == '2020']\ndf.shape","metadata":{"papermill":{"duration":11.627335,"end_time":"2022-03-13T09:43:19.513809","exception":false,"start_time":"2022-03-13T09:43:07.886474","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:08.016745Z","iopub.execute_input":"2022-03-29T01:30:08.017379Z","iopub.status.idle":"2022-03-29T01:30:18.556256Z","shell.execute_reply.started":"2022-03-29T01:30:08.017326Z","shell.execute_reply":"2022-03-29T01:30:18.555455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_user = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\ndf_test_user.shape","metadata":{"papermill":{"duration":5.11593,"end_time":"2022-03-13T09:43:24.675102","exception":false,"start_time":"2022-03-13T09:43:19.559172","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:18.557403Z","iopub.execute_input":"2022-03-29T01:30:18.55762Z","iopub.status.idle":"2022-03-29T01:30:23.703442Z","shell.execute_reply.started":"2022-03-29T01:30:18.557595Z","shell.execute_reply":"2022-03-29T01:30:23.702535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find inactive user","metadata":{"papermill":{"duration":0.046827,"end_time":"2022-03-13T09:43:24.770393","exception":false,"start_time":"2022-03-13T09:43:24.723566","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"First, let count number of transaction in each month for all users","metadata":{"papermill":{"duration":0.046506,"end_time":"2022-03-13T09:43:24.865791","exception":false,"start_time":"2022-03-13T09:43:24.819285","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_month_avg_item_per_u = df.groupby(['customer_id', 'month'])['price'].count().unstack().reset_index()\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":8.493684,"end_time":"2022-03-13T09:43:33.408031","exception":false,"start_time":"2022-03-13T09:43:24.914347","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:23.705584Z","iopub.execute_input":"2022-03-29T01:30:23.705824Z","iopub.status.idle":"2022-03-29T01:30:31.530403Z","shell.execute_reply.started":"2022-03-29T01:30:23.705796Z","shell.execute_reply":"2022-03-29T01:30:31.529551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then merge with test data. Test data has more rows than our transaction data. It means we have some users who don't have any transactions in 2020 in test data. Let check how many users like it","metadata":{"papermill":{"duration":0.045585,"end_time":"2022-03-13T09:43:33.499791","exception":false,"start_time":"2022-03-13T09:43:33.454206","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_month_avg_item_per_u = pd.merge(df_month_avg_item_per_u, df_test_user[['customer_id']], on='customer_id', how='outer')\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":1.756256,"end_time":"2022-03-13T09:43:35.304349","exception":false,"start_time":"2022-03-13T09:43:33.548093","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:31.532037Z","iopub.execute_input":"2022-03-29T01:30:31.532503Z","iopub.status.idle":"2022-03-29T01:30:33.293461Z","shell.execute_reply.started":"2022-03-29T01:30:31.53246Z","shell.execute_reply":"2022-03-29T01:30:33.292551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_month_avg_item_per_u['num_missing_months'] = df_month_avg_item_per_u.isnull().sum(axis=1)\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":0.31912,"end_time":"2022-03-13T09:43:35.670263","exception":false,"start_time":"2022-03-13T09:43:35.351143","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:33.294782Z","iopub.execute_input":"2022-03-29T01:30:33.295055Z","iopub.status.idle":"2022-03-29T01:30:33.563152Z","shell.execute_reply.started":"2022-03-29T01:30:33.295022Z","shell.execute_reply":"2022-03-29T01:30:33.562071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**num_missing_months=9 means users don't have any transactions in 2020(9 months of 2020). There is 37% users with this condition in test data**","metadata":{"papermill":{"duration":0.047182,"end_time":"2022-03-13T09:43:35.763951","exception":false,"start_time":"2022-03-13T09:43:35.716769","status":"completed"},"tags":[]}},{"cell_type":"code","source":"num_missing_year = len(df_month_avg_item_per_u[df_month_avg_item_per_u['num_missing_months'] == 9])\nprint(num_missing_year)\nprint(num_missing_year/len(df_test_user))","metadata":{"papermill":{"duration":0.112139,"end_time":"2022-03-13T09:43:35.923128","exception":false,"start_time":"2022-03-13T09:43:35.810989","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:33.564595Z","iopub.execute_input":"2022-03-29T01:30:33.564834Z","iopub.status.idle":"2022-03-29T01:30:33.619084Z","shell.execute_reply.started":"2022-03-29T01:30:33.564808Z","shell.execute_reply":"2022-03-29T01:30:33.618154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_month_avg_item_per_u = df_month_avg_item_per_u.fillna(0)\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":0.408738,"end_time":"2022-03-13T09:43:36.380966","exception":false,"start_time":"2022-03-13T09:43:35.972228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:33.620456Z","iopub.execute_input":"2022-03-29T01:30:33.620764Z","iopub.status.idle":"2022-03-29T01:30:33.964918Z","shell.execute_reply.started":"2022-03-29T01:30:33.620724Z","shell.execute_reply":"2022-03-29T01:30:33.964077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inactive users with more than 3 consecutive months will be still masked as 3**","metadata":{"papermill":{"duration":0.050434,"end_time":"2022-03-13T09:43:36.480361","exception":false,"start_time":"2022-03-13T09:43:36.429927","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def cal_inactive_months(x):\n    if x['09'] > 0:\n        return 0\n    elif x['09'] == 0 and x['08'] > 0:\n        return 1\n    elif x['09'] == 0 and x['08'] == 0 and x['07'] > 0:\n        return 2\n    elif x['09'] == 0 and x['08'] == 0 and x['07'] == 0:\n        return 3\n    else:\n        return 4\n\ndf_month_avg_item_per_u['lastest_inactive_months'] = df_month_avg_item_per_u[\n    df_month_avg_item_per_u.columns.difference(['customer_id', 'num_missing_months'])].apply(\n    lambda x: cal_inactive_months(x), axis=1)\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":66.088592,"end_time":"2022-03-13T09:44:42.618071","exception":false,"start_time":"2022-03-13T09:43:36.529479","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:30:33.966421Z","iopub.execute_input":"2022-03-29T01:30:33.967045Z","iopub.status.idle":"2022-03-29T01:31:39.155543Z","shell.execute_reply.started":"2022-03-29T01:30:33.966968Z","shell.execute_reply":"2022-03-29T01:31:39.154721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**In below cell, we see that 50% of users disappers in 8 or more months before reactive. It is another challenge in our data**","metadata":{"papermill":{"duration":0.051176,"end_time":"2022-03-13T09:44:42.718861","exception":false,"start_time":"2022-03-13T09:44:42.667685","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(df_month_avg_item_per_u.num_missing_months.value_counts())\nprint(df_month_avg_item_per_u.num_missing_months.describe())\ndf_month_avg_item_per_u.num_missing_months.hist()","metadata":{"papermill":{"duration":0.445556,"end_time":"2022-03-13T09:44:43.213989","exception":false,"start_time":"2022-03-13T09:44:42.768433","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:39.158382Z","iopub.execute_input":"2022-03-29T01:31:39.158605Z","iopub.status.idle":"2022-03-29T01:31:39.704859Z","shell.execute_reply.started":"2022-03-29T01:31:39.15858Z","shell.execute_reply":"2022-03-29T01:31:39.704252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**In following cell, we see that 63% of users disappeared in recent 3 months (Sep, Aug, July) before reappear in testdata.**","metadata":{"papermill":{"duration":0.050471,"end_time":"2022-03-13T09:44:43.314756","exception":false,"start_time":"2022-03-13T09:44:43.264285","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(df_month_avg_item_per_u.lastest_inactive_months.value_counts())\nprint(df_month_avg_item_per_u.lastest_inactive_months.describe())\ndf_month_avg_item_per_u.lastest_inactive_months.hist()","metadata":{"papermill":{"duration":0.533663,"end_time":"2022-03-13T09:44:43.899305","exception":false,"start_time":"2022-03-13T09:44:43.365642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:39.705965Z","iopub.execute_input":"2022-03-29T01:31:39.706319Z","iopub.status.idle":"2022-03-29T01:31:40.015014Z","shell.execute_reply.started":"2022-03-29T01:31:39.706289Z","shell.execute_reply":"2022-03-29T01:31:40.014118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Missing 3 months\")\nnum_missing_3months = len(df_month_avg_item_per_u[df_month_avg_item_per_u['lastest_inactive_months'] == 3])\nprint(num_missing_3months)\nprint(num_missing_3months/len(df_test_user))\nprint(\"Missing 2 months\")\nnum_missing_2months = len(df_month_avg_item_per_u[df_month_avg_item_per_u['lastest_inactive_months'] == 2])\nprint(num_missing_2months)\nprint(num_missing_2months/len(df_test_user))\nprint(\"Missing 1 months\")\nnum_missing_1months = len(df_month_avg_item_per_u[df_month_avg_item_per_u['lastest_inactive_months'] == 1])\nprint(num_missing_1months)\nprint(num_missing_1months/len(df_test_user))","metadata":{"papermill":{"duration":0.231068,"end_time":"2022-03-13T09:44:44.184117","exception":false,"start_time":"2022-03-13T09:44:43.953049","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:40.016184Z","iopub.execute_input":"2022-03-29T01:31:40.016416Z","iopub.status.idle":"2022-03-29T01:31:40.193237Z","shell.execute_reply.started":"2022-03-29T01:31:40.016384Z","shell.execute_reply":"2022-03-29T01:31:40.19219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a dataframe for inactive user, in order to merge with other information about user","metadata":{"papermill":{"duration":0.0523,"end_time":"2022-03-13T09:44:44.290041","exception":false,"start_time":"2022-03-13T09:44:44.237741","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_month_avg_item_per_u['active_status'] = 'active'\ndf_month_avg_item_per_u.loc[(df_month_avg_item_per_u.num_missing_months == 9),'active_status']='inactive_in_year'\ndf_month_avg_item_per_u.loc[(df_month_avg_item_per_u.num_missing_months < 9) &\n                            (df_month_avg_item_per_u.lastest_inactive_months == 3),\n                            'active_status']='inactive_in_3_months_or_more'\ndf_month_avg_item_per_u.loc[\n    (df_month_avg_item_per_u.lastest_inactive_months == 2),'active_status']='inactive_in_2_months'\ndf_month_avg_item_per_u.loc[\n    (df_month_avg_item_per_u.lastest_inactive_months == 1),'active_status']='inactive_in_1_month'\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":0.217468,"end_time":"2022-03-13T09:44:44.560965","exception":false,"start_time":"2022-03-13T09:44:44.343497","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:40.194663Z","iopub.execute_input":"2022-03-29T01:31:40.195093Z","iopub.status.idle":"2022-03-29T01:31:40.356896Z","shell.execute_reply.started":"2022-03-29T01:31:40.195046Z","shell.execute_reply":"2022-03-29T01:31:40.356012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_active_user = df_month_avg_item_per_u[['customer_id', 'num_missing_months', 'lastest_inactive_months', 'active_status']].copy()\ndf_active_user","metadata":{"papermill":{"duration":0.347145,"end_time":"2022-03-13T09:44:44.96257","exception":false,"start_time":"2022-03-13T09:44:44.615425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:40.358269Z","iopub.execute_input":"2022-03-29T01:31:40.358484Z","iopub.status.idle":"2022-03-29T01:31:40.579081Z","shell.execute_reply.started":"2022-03-29T01:31:40.358458Z","shell.execute_reply":"2022-03-29T01:31:40.578408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find coldstart customer","metadata":{"papermill":{"duration":0.055697,"end_time":"2022-03-13T09:44:45.082149","exception":false,"start_time":"2022-03-13T09:44:45.026452","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**First, count number of transaction. We will mask users with number of transactions <= 10 are cold start user. They are users with too small data for correct recommendation**","metadata":{"papermill":{"duration":0.053204,"end_time":"2022-03-13T09:44:45.19057","exception":false,"start_time":"2022-03-13T09:44:45.137366","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_avg_item_per_u = df.groupby(['customer_id'])['price'].count().reset_index()\ndf_avg_item_per_u.columns = ['customer_id', 'num_transactions']\ndf_avg_item_per_u","metadata":{"papermill":{"duration":5.078477,"end_time":"2022-03-13T09:44:50.323569","exception":false,"start_time":"2022-03-13T09:44:45.245092","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:40.580055Z","iopub.execute_input":"2022-03-29T01:31:40.580666Z","iopub.status.idle":"2022-03-29T01:31:45.6041Z","shell.execute_reply.started":"2022-03-29T01:31:40.580631Z","shell.execute_reply":"2022-03-29T01:31:45.603018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In test data, we have some users who dont have any transactions in 2020. Let add it into our dataframe, and label their number of transaction as 0","metadata":{"papermill":{"duration":0.058409,"end_time":"2022-03-13T09:44:50.455035","exception":false,"start_time":"2022-03-13T09:44:50.396626","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_avg_item_per_u = pd.merge(df_avg_item_per_u, df_test_user[['customer_id']], on='customer_id', how='outer')\ndf_avg_item_per_u = df_avg_item_per_u.fillna(0)\ndf_avg_item_per_u","metadata":{"papermill":{"duration":1.936234,"end_time":"2022-03-13T09:44:52.455043","exception":false,"start_time":"2022-03-13T09:44:50.518809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:45.605488Z","iopub.execute_input":"2022-03-29T01:31:45.606069Z","iopub.status.idle":"2022-03-29T01:31:47.466787Z","shell.execute_reply.started":"2022-03-29T01:31:45.606024Z","shell.execute_reply":"2022-03-29T01:31:47.465996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**In below plot, we see that most of users have small number of transactions**","metadata":{"papermill":{"duration":0.055309,"end_time":"2022-03-13T09:44:52.565236","exception":false,"start_time":"2022-03-13T09:44:52.509927","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_avg_item_per_u.num_transactions.hist(bins=100)\nplt.show()\nplt.close()\ndf_avg_item_per_u.boxplot('num_transactions')\nplt.show()\nplt.close()","metadata":{"papermill":{"duration":3.580931,"end_time":"2022-03-13T09:44:56.202656","exception":false,"start_time":"2022-03-13T09:44:52.621725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:47.468068Z","iopub.execute_input":"2022-03-29T01:31:47.468359Z","iopub.status.idle":"2022-03-29T01:31:50.976718Z","shell.execute_reply.started":"2022-03-29T01:31:47.468318Z","shell.execute_reply":"2022-03-29T01:31:50.975812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_avg_item_per_u.num_transactions.value_counts(bins=[-1, 0, 10, 100, 1000])","metadata":{"papermill":{"duration":0.123725,"end_time":"2022-03-13T09:44:56.383775","exception":false,"start_time":"2022-03-13T09:44:56.26005","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:50.977827Z","iopub.execute_input":"2022-03-29T01:31:50.978048Z","iopub.status.idle":"2022-03-29T01:31:51.035901Z","shell.execute_reply.started":"2022-03-29T01:31:50.978022Z","shell.execute_reply":"2022-03-29T01:31:51.035065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_avg_item_per_u.num_transactions.describe()","metadata":{"papermill":{"duration":0.116979,"end_time":"2022-03-13T09:44:56.559065","exception":false,"start_time":"2022-03-13T09:44:56.442086","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:51.03731Z","iopub.execute_input":"2022-03-29T01:31:51.038199Z","iopub.status.idle":"2022-03-29T01:31:51.087022Z","shell.execute_reply.started":"2022-03-29T01:31:51.038153Z","shell.execute_reply":"2022-03-29T01:31:51.086355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**we mask users with num transaction < 10 as cold start user**","metadata":{"papermill":{"duration":0.057947,"end_time":"2022-03-13T09:44:56.677565","exception":false,"start_time":"2022-03-13T09:44:56.619618","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_avg_item_per_u['cold_start_status'] = 'cold_start'\ndf_avg_item_per_u.loc[(df_avg_item_per_u.num_transactions >= 10),'cold_start_status']='non_cold_start'\ndf_coldstart_user = df_avg_item_per_u.copy()\ndf_coldstart_user","metadata":{"papermill":{"duration":0.304245,"end_time":"2022-03-13T09:44:57.039775","exception":false,"start_time":"2022-03-13T09:44:56.73553","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:51.088124Z","iopub.execute_input":"2022-03-29T01:31:51.088464Z","iopub.status.idle":"2022-03-29T01:31:51.334362Z","shell.execute_reply.started":"2022-03-29T01:31:51.088437Z","shell.execute_reply":"2022-03-29T01:31:51.333453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find about frequent transaction of user in month ","metadata":{"papermill":{"duration":0.057706,"end_time":"2022-03-13T09:44:57.15593","exception":false,"start_time":"2022-03-13T09:44:57.098224","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_month_avg_item_per_u = df.groupby(['customer_id', 'month'])['price'].count().unstack().reset_index()\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":8.700719,"end_time":"2022-03-13T09:45:05.914923","exception":false,"start_time":"2022-03-13T09:44:57.214204","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:51.33575Z","iopub.execute_input":"2022-03-29T01:31:51.336037Z","iopub.status.idle":"2022-03-29T01:31:59.517923Z","shell.execute_reply.started":"2022-03-29T01:31:51.336004Z","shell.execute_reply":"2022-03-29T01:31:59.516952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_active_month(x):\n    float_x = x.values[1:].astype(float)\n    return float_x[~np.isnan(float_x)]\ndf_month_avg_item_per_u['transactions_in_active_month'] = df_month_avg_item_per_u.apply(\n    lambda x: find_active_month(x), axis=1)\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":14.770339,"end_time":"2022-03-13T09:45:20.752055","exception":false,"start_time":"2022-03-13T09:45:05.981716","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:31:59.519427Z","iopub.execute_input":"2022-03-29T01:31:59.519731Z","iopub.status.idle":"2022-03-29T01:32:13.817798Z","shell.execute_reply.started":"2022-03-29T01:31:59.519689Z","shell.execute_reply":"2022-03-29T01:32:13.816989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_month_avg_item_per_u['mean_transactions_in_active_month'] = df_month_avg_item_per_u.apply(\n    lambda x: x['transactions_in_active_month'].mean(), axis=1)\ndf_month_avg_item_per_u","metadata":{"papermill":{"duration":23.882654,"end_time":"2022-03-13T09:45:44.696263","exception":false,"start_time":"2022-03-13T09:45:20.813609","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:13.819152Z","iopub.execute_input":"2022-03-29T01:32:13.819719Z","iopub.status.idle":"2022-03-29T01:32:37.187381Z","shell.execute_reply.started":"2022-03-29T01:32:13.819679Z","shell.execute_reply":"2022-03-29T01:32:37.186536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In average, each user only buy 4 items in a month/1 item in a week. It is another challenge","metadata":{"papermill":{"duration":0.062587,"end_time":"2022-03-13T09:45:44.820742","exception":false,"start_time":"2022-03-13T09:45:44.758155","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(df_month_avg_item_per_u.mean_transactions_in_active_month.describe())\ndf_month_avg_item_per_u.mean_transactions_in_active_month.hist(bins=100)","metadata":{"papermill":{"duration":0.483043,"end_time":"2022-03-13T09:45:45.364195","exception":false,"start_time":"2022-03-13T09:45:44.881152","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:37.188799Z","iopub.execute_input":"2022-03-29T01:32:37.18943Z","iopub.status.idle":"2022-03-29T01:32:37.649524Z","shell.execute_reply.started":"2022-03-29T01:32:37.189393Z","shell.execute_reply":"2022-03-29T01:32:37.648593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create dataframe for all metadata for user: active status/cold start status","metadata":{"papermill":{"duration":0.063849,"end_time":"2022-03-13T09:45:45.491673","exception":false,"start_time":"2022-03-13T09:45:45.427824","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_transaction_frequent = df_month_avg_item_per_u[['customer_id', 'mean_transactions_in_active_month']].copy()\ndf_transaction_frequent","metadata":{"papermill":{"duration":0.187279,"end_time":"2022-03-13T09:45:45.74395","exception":false,"start_time":"2022-03-13T09:45:45.556671","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:37.65109Z","iopub.execute_input":"2022-03-29T01:32:37.651352Z","iopub.status.idle":"2022-03-29T01:32:37.774798Z","shell.execute_reply.started":"2022-03-29T01:32:37.651321Z","shell.execute_reply":"2022-03-29T01:32:37.774023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.merge(df_active_user, df_coldstart_user, on='customer_id', how='outer')\nresult = pd.merge(result, df_transaction_frequent, on='customer_id', how='outer')\nresult","metadata":{"papermill":{"duration":3.013715,"end_time":"2022-03-13T09:45:48.827404","exception":false,"start_time":"2022-03-13T09:45:45.813689","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:37.776091Z","iopub.execute_input":"2022-03-29T01:32:37.776309Z","iopub.status.idle":"2022-03-29T01:32:40.815794Z","shell.execute_reply.started":"2022-03-29T01:32:37.776282Z","shell.execute_reply":"2022-03-29T01:32:40.81494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result[(result.active_status == 'active') & (result.cold_start_status == 'non_cold_start')].shape","metadata":{"papermill":{"duration":0.655124,"end_time":"2022-03-13T09:45:49.5471","exception":false,"start_time":"2022-03-13T09:45:48.891976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:40.817696Z","iopub.execute_input":"2022-03-29T01:32:40.818064Z","iopub.status.idle":"2022-03-29T01:32:41.406863Z","shell.execute_reply.started":"2022-03-29T01:32:40.818018Z","shell.execute_reply":"2022-03-29T01:32:41.406024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv('metadata_customer_id_fold1.csv', index=False)","metadata":{"papermill":{"duration":10.317918,"end_time":"2022-03-13T09:45:59.93075","exception":false,"start_time":"2022-03-13T09:45:49.612832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:41.410358Z","iopub.execute_input":"2022-03-29T01:32:41.410579Z","iopub.status.idle":"2022-03-29T01:32:51.472138Z","shell.execute_reply.started":"2022-03-29T01:32:41.410553Z","shell.execute_reply":"2022-03-29T01:32:51.471351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.shape","metadata":{"papermill":{"duration":0.074167,"end_time":"2022-03-13T09:46:00.070685","exception":false,"start_time":"2022-03-13T09:45:59.996518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:51.473378Z","iopub.execute_input":"2022-03-29T01:32:51.473621Z","iopub.status.idle":"2022-03-29T01:32:51.47969Z","shell.execute_reply.started":"2022-03-29T01:32:51.473591Z","shell.execute_reply":"2022-03-29T01:32:51.479123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"active & non cold start user: {len(result[(result.active_status == 'active') & (result.cold_start_status == 'non_cold_start')])/len(result)*100 :.2f}%\")","metadata":{"papermill":{"duration":0.074445,"end_time":"2022-03-13T09:46:00.210318","exception":false,"start_time":"2022-03-13T09:46:00.135873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-29T01:32:51.48059Z","iopub.execute_input":"2022-03-29T01:32:51.481222Z","iopub.status.idle":"2022-03-29T01:32:52.063879Z","shell.execute_reply.started":"2022-03-29T01:32:51.481186Z","shell.execute_reply":"2022-03-29T01:32:52.06292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation for This Strategy","metadata":{}},{"cell_type":"code","source":"valid_df = pd.read_csv(f'../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\nvalid_df['t_dat'] = pd.to_datetime(valid_df['t_dat'])\nvalid_df = valid_df[valid_df['t_dat'] >= pd.to_datetime('2020-09-16')]\nvalid_user = valid_df[['customer_id']].drop_duplicates()\nvalid_user = valid_user.merge(result, how='left')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T01:35:56.186182Z","iopub.execute_input":"2022-03-29T01:35:56.18675Z","iopub.status.idle":"2022-03-29T01:36:46.620301Z","shell.execute_reply.started":"2022-03-29T01:35:56.186716Z","shell.execute_reply":"2022-03-29T01:36:46.618728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.float_format = '{:.2f}'.format\nvalid_user_dist = pd.DataFrame()\nfor c in ['lastest_inactive_months', 'active_status', 'cold_start_status']:\n    tmp = (valid_user[c].value_counts().sort_index()/len(valid_user)*100).reset_index()\n    tmp['column'] = c\n    tmp = tmp[tmp.columns[[2,0,1]]]\n    tmp.columns = ['column', 'value', 'percent']\n    tmp = tmp.set_index(['column', 'value'])\n    valid_user_dist = pd.concat([valid_user_dist, tmp], axis=0)\nvalid_user_dist","metadata":{"execution":{"iopub.status.busy":"2022-03-29T01:37:17.180893Z","iopub.execute_input":"2022-03-29T01:37:17.182022Z","iopub.status.idle":"2022-03-29T01:37:17.246505Z","shell.execute_reply.started":"2022-03-29T01:37:17.181951Z","shell.execute_reply":"2022-03-29T01:37:17.24556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"active & non cold start user (by all users who bought any item in valid term): \\\n    {len(valid_user[(valid_user['active_status']=='active') & (valid_user['cold_start_status']=='non_cold_start')])/len(valid_user)*100 :.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2022-03-29T01:37:17.667337Z","iopub.execute_input":"2022-03-29T01:37:17.667634Z","iopub.status.idle":"2022-03-29T01:37:17.70474Z","shell.execute_reply.started":"2022-03-29T01:37:17.667603Z","shell.execute_reply":"2022-03-29T01:37:17.703898Z"},"trusted":true},"execution_count":null,"outputs":[]}]}