{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport os\nimport gc \nfrom datetime import datetime\nfrom collections import Counter\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-20T06:23:39.912738Z","iopub.execute_input":"2022-03-20T06:23:39.913009Z","iopub.status.idle":"2022-03-20T06:23:41.081550Z","shell.execute_reply.started":"2022-03-20T06:23:39.912942Z","shell.execute_reply":"2022-03-20T06:23:41.080654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:23:41.083616Z","iopub.execute_input":"2022-03-20T06:23:41.084312Z","iopub.status.idle":"2022-03-20T06:24:51.225805Z","shell.execute_reply.started":"2022-03-20T06:23:41.084268Z","shell.execute_reply":"2022-03-20T06:24:51.224903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:24:51.227011Z","iopub.execute_input":"2022-03-20T06:24:51.227305Z","iopub.status.idle":"2022-03-20T06:24:51.256996Z","shell.execute_reply.started":"2022-03-20T06:24:51.227267Z","shell.execute_reply":"2022-03-20T06:24:51.255742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:24:51.259685Z","iopub.execute_input":"2022-03-20T06:24:51.259963Z","iopub.status.idle":"2022-03-20T06:24:51.267709Z","shell.execute_reply.started":"2022-03-20T06:24:51.259932Z","shell.execute_reply":"2022-03-20T06:24:51.266568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['t_dat'].max()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:24:51.269666Z","iopub.execute_input":"2022-03-20T06:24:51.270146Z","iopub.status.idle":"2022-03-20T06:24:55.685537Z","shell.execute_reply.started":"2022-03-20T06:24:51.270106Z","shell.execute_reply":"2022-03-20T06:24:55.684489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['t_dat'].min()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:24:55.686800Z","iopub.execute_input":"2022-03-20T06:24:55.687028Z","iopub.status.idle":"2022-03-20T06:25:00.381920Z","shell.execute_reply.started":"2022-03-20T06:24:55.687000Z","shell.execute_reply":"2022-03-20T06:25:00.381135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (0) Split dataset by transaction year","metadata":{}},{"cell_type":"code","source":"transactions_2020 = transactions[transactions['t_dat']>='2020-01-01']\ntransactions.drop(index=transactions_2020.index,inplace=True)\n\ntransactions_2019 = transactions[transactions['t_dat']>='2019-01-01']\ntransactions.drop(index=transactions_2019.index,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:00.384398Z","iopub.execute_input":"2022-03-20T06:25:00.384681Z","iopub.status.idle":"2022-03-20T06:25:20.795954Z","shell.execute_reply.started":"2022-03-20T06:25:00.384646Z","shell.execute_reply":"2022-03-20T06:25:20.794891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018 = transactions.copy()\ndel transactions","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:20.797346Z","iopub.execute_input":"2022-03-20T06:25:20.797535Z","iopub.status.idle":"2022-03-20T06:25:20.881737Z","shell.execute_reply.started":"2022-03-20T06:25:20.797509Z","shell.execute_reply":"2022-03-20T06:25:20.880861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:20.883046Z","iopub.execute_input":"2022-03-20T06:25:20.883271Z","iopub.status.idle":"2022-03-20T06:25:21.046717Z","shell.execute_reply.started":"2022-03-20T06:25:20.883245Z","shell.execute_reply":"2022-03-20T06:25:21.045925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2019['t_dat'].max(),transactions_2019['t_dat'].min()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:21.049908Z","iopub.execute_input":"2022-03-20T06:25:21.050318Z","iopub.status.idle":"2022-03-20T06:25:25.752225Z","shell.execute_reply.started":"2022-03-20T06:25:21.050284Z","shell.execute_reply":"2022-03-20T06:25:25.751133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2020['t_dat'].max(),transactions_2020['t_dat'].min()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:25.753602Z","iopub.execute_input":"2022-03-20T06:25:25.753831Z","iopub.status.idle":"2022-03-20T06:25:28.889066Z","shell.execute_reply.started":"2022-03-20T06:25:25.753804Z","shell.execute_reply":"2022-03-20T06:25:28.888152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018['t_dat'].max(),transactions_2018['t_dat'].min()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:28.890331Z","iopub.execute_input":"2022-03-20T06:25:28.890604Z","iopub.status.idle":"2022-03-20T06:25:30.158466Z","shell.execute_reply.started":"2022-03-20T06:25:28.890566Z","shell.execute_reply":"2022-03-20T06:25:30.157614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018['t_dat'] = pd.DatetimeIndex(transactions_2018['t_dat'])\ntransactions_2020['t_dat'] = pd.DatetimeIndex(transactions_2020['t_dat'])\ntransactions_2019['t_dat'] = pd.DatetimeIndex(transactions_2019['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:30.159641Z","iopub.execute_input":"2022-03-20T06:25:30.159880Z","iopub.status.idle":"2022-03-20T06:25:42.454428Z","shell.execute_reply.started":"2022-03-20T06:25:30.159853Z","shell.execute_reply":"2022-03-20T06:25:42.453476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018.reset_index(drop=True,inplace=True)\ntransactions_2019.reset_index(drop=True,inplace=True)\ntransactions_2020.reset_index(drop=True,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:42.455615Z","iopub.execute_input":"2022-03-20T06:25:42.455886Z","iopub.status.idle":"2022-03-20T06:25:42.462788Z","shell.execute_reply.started":"2022-03-20T06:25:42.455850Z","shell.execute_reply":"2022-03-20T06:25:42.461586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (1) Create RFM Features","metadata":{}},{"cell_type":"code","source":"class RFM():\n    def __init__(self):\n        self.recency_df = None\n        self.frequency_df = None\n        self.monetary_df = None\n        self.rfm_segmentation = None\n        \n    def RScore(self,x,p,d):\n        if x <= d[p][0.25]:\n            return 4\n        elif x <= d[p][0.50]:\n            return 3\n        elif x <= d[p][0.75]: \n            return 2\n        else:\n            return 1\n    \n    def FMScore(self,x,p,d):\n        if x <= d[p][0.25]:\n            return 1\n        elif x <= d[p][0.50]:\n            return 2\n        elif x <= d[p][0.75]: \n            return 3\n        else:\n            return 4\n    \n    def fit(self,data,cusomer_id_col,date_column,amount_col,trans_id_col='index'):\n        data = data.reset_index().copy()\n        data[amount_col] = data[amount_col].astype(float)\n        now = datetime.now()\n        \n        self.recency_df = data.groupby(by=cusomer_id_col, as_index=False)[date_column].max()\n        self.recency_df.columns = ['CustomerID','LastPurshaceDate']\n        self.recency_df['Recency'] = self.recency_df['LastPurshaceDate'].apply(lambda x: (now - x).days)\n        \n        self.frequency_df = data.groupby(by=[cusomer_id_col], as_index=False)[trans_id_col].count()\n        self.frequency_df.columns = ['CustomerID','Frequency']\n        \n        self.monetary_df = data.groupby(by=cusomer_id_col,as_index=False).agg({amount_col: 'sum'})\n        self.monetary_df.columns = ['CustomerID','Monetary']\n        \n        temp_df = self.recency_df.merge(self.frequency_df,on='CustomerID')\n        rfm_df = temp_df.merge(self.monetary_df,on='CustomerID')\n        rfm_df.set_index('CustomerID',inplace=True)\n        \n        self.quantiles = rfm_df.quantile(q=[0.25,0.5,0.75])\n        \n        self.rfm_segmentation = rfm_df\n        self.rfm_segmentation['R_Quartile'] = self.rfm_segmentation['Recency'].apply(self.RScore, args=('Recency',self.quantiles,))\n        self.rfm_segmentation['F_Quartile'] = self.rfm_segmentation['Frequency'].apply(self.FMScore, args=('Frequency',self.quantiles,))\n        self.rfm_segmentation['M_Quartile'] = self.rfm_segmentation['Monetary'].apply(self.FMScore, args=('Monetary',self.quantiles,))\n        \n        self.rfm_segmentation['RFMScore'] = self.rfm_segmentation.R_Quartile.map(str) \\\n                            + self.rfm_segmentation.F_Quartile.map(str) \\\n                            + self.rfm_segmentation.M_Quartile.map(str)\n        return self","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:42.464632Z","iopub.execute_input":"2022-03-20T06:25:42.464904Z","iopub.status.idle":"2022-03-20T06:25:42.486149Z","shell.execute_reply.started":"2022-03-20T06:25:42.464874Z","shell.execute_reply":"2022-03-20T06:25:42.485493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfm_2018 = RFM()\nrfm_2019 = RFM()\nrfm_2020 = RFM()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:42.486996Z","iopub.execute_input":"2022-03-20T06:25:42.487219Z","iopub.status.idle":"2022-03-20T06:25:42.507372Z","shell.execute_reply.started":"2022-03-20T06:25:42.487192Z","shell.execute_reply":"2022-03-20T06:25:42.506529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time rfm_2018.fit(transactions_2018,cusomer_id_col='customer_id',date_column='t_dat',amount_col='price')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:25:42.508746Z","iopub.execute_input":"2022-03-20T06:25:42.509158Z","iopub.status.idle":"2022-03-20T06:26:40.396327Z","shell.execute_reply.started":"2022-03-20T06:25:42.509131Z","shell.execute_reply":"2022-03-20T06:26:40.395536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time rfm_2019.fit(transactions_2019,cusomer_id_col='customer_id',date_column='t_dat',amount_col='price')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:26:40.397461Z","iopub.execute_input":"2022-03-20T06:26:40.397627Z","iopub.status.idle":"2022-03-20T06:28:04.705907Z","shell.execute_reply.started":"2022-03-20T06:26:40.397606Z","shell.execute_reply":"2022-03-20T06:28:04.704803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time rfm_2020.fit(transactions_2020,cusomer_id_col='customer_id',date_column='t_dat',amount_col='price')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:28:04.707547Z","iopub.execute_input":"2022-03-20T06:28:04.707756Z","iopub.status.idle":"2022-03-20T06:29:07.774221Z","shell.execute_reply.started":"2022-03-20T06:28:04.707734Z","shell.execute_reply":"2022-03-20T06:29:07.773408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen = ({key:value} for key,value in enumerate(range(10)))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:07.775382Z","iopub.execute_input":"2022-03-20T06:29:07.775604Z","iopub.status.idle":"2022-03-20T06:29:07.782280Z","shell.execute_reply.started":"2022-03-20T06:29:07.775572Z","shell.execute_reply":"2022-03-20T06:29:07.781145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfm_segmentation_2020 = rfm_2020.rfm_segmentation\nrfm_segmentation_2019 = rfm_2019.rfm_segmentation\nrfm_segmentation_2018 = rfm_2018.rfm_segmentation","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:07.783931Z","iopub.execute_input":"2022-03-20T06:29:07.784582Z","iopub.status.idle":"2022-03-20T06:29:07.794296Z","shell.execute_reply.started":"2022-03-20T06:29:07.784548Z","shell.execute_reply":"2022-03-20T06:29:07.792704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018_copy = transactions_2018.set_index('customer_id')\ntransactions_2019_copy = transactions_2019.set_index('customer_id')\ntransactions_2020_copy = transactions_2020.set_index('customer_id')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:07.795448Z","iopub.execute_input":"2022-03-20T06:29:07.795657Z","iopub.status.idle":"2022-03-20T06:29:08.255607Z","shell.execute_reply.started":"2022-03-20T06:29:07.795630Z","shell.execute_reply":"2022-03-20T06:29:08.254721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_2018_copy.shape[0]==transactions_2018.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:08.257029Z","iopub.execute_input":"2022-03-20T06:29:08.257257Z","iopub.status.idle":"2022-03-20T06:29:08.264221Z","shell.execute_reply.started":"2022-03-20T06:29:08.257230Z","shell.execute_reply":"2022-03-20T06:29:08.263071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del transactions_2018\ndel transactions_2019\ndel transactions_2020","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:08.265429Z","iopub.execute_input":"2022-03-20T06:29:08.265664Z","iopub.status.idle":"2022-03-20T06:29:08.430573Z","shell.execute_reply.started":"2022-03-20T06:29:08.265637Z","shell.execute_reply":"2022-03-20T06:29:08.428999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (2) Get last purchased item","metadata":{}},{"cell_type":"code","source":"def get_last_purchased_item(transactions_df,last_perchased_date):\n#     temp_df = transactions_df.join(last_perchased_date)[['article_id']]\n    return transactions_df.join(last_perchased_date)\n#     return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:08.432444Z","iopub.execute_input":"2022-03-20T06:29:08.433114Z","iopub.status.idle":"2022-03-20T06:29:08.438268Z","shell.execute_reply.started":"2022-03-20T06:29:08.433063Z","shell.execute_reply":"2022-03-20T06:29:08.436575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_puchased_item_2018_test =  get_last_purchased_item(transactions_2018_copy,rfm_segmentation_2018[['LastPurshaceDate']])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:08.440138Z","iopub.execute_input":"2022-03-20T06:29:08.440413Z","iopub.status.idle":"2022-03-20T06:29:13.326321Z","shell.execute_reply.started":"2022-03-20T06:29:08.440384Z","shell.execute_reply":"2022-03-20T06:29:13.325390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_puchased_item_2018_test.sample(10000).reset_index().groupby('index').agg({\"article_id\":'count'}).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:13.327563Z","iopub.execute_input":"2022-03-20T06:29:13.328031Z","iopub.status.idle":"2022-03-20T06:29:13.941664Z","shell.execute_reply.started":"2022-03-20T06:29:13.328002Z","shell.execute_reply":"2022-03-20T06:29:13.940667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_puchased_item_2018_test.sample(10000).reset_index().groupby('index').agg({\"article_id\":'count'}).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:13.945484Z","iopub.execute_input":"2022-03-20T06:29:13.945659Z","iopub.status.idle":"2022-03-20T06:29:14.185590Z","shell.execute_reply.started":"2022-03-20T06:29:13.945638Z","shell.execute_reply":"2022-03-20T06:29:14.184696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_puchased_item_2018_test.sample(10000).reset_index().groupby('index').agg({\"article_id\":'count'}).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:14.186718Z","iopub.execute_input":"2022-03-20T06:29:14.187466Z","iopub.status.idle":"2022-03-20T06:29:14.820129Z","shell.execute_reply.started":"2022-03-20T06:29:14.187430Z","shell.execute_reply":"2022-03-20T06:29:14.818876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_last_purchased_item(transactions_df,last_perchased_date,year):\n    temp_df = transactions_df.join(last_perchased_date)[['article_id']]\n    temp_df = temp_df[~temp_df.index.duplicated(keep='first')]\n    temp_df.columns = [f\"last_purchesed_item_{str(year)}\"]\n    return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:14.822215Z","iopub.execute_input":"2022-03-20T06:29:14.822519Z","iopub.status.idle":"2022-03-20T06:29:14.828095Z","shell.execute_reply.started":"2022-03-20T06:29:14.822482Z","shell.execute_reply":"2022-03-20T06:29:14.827045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nlast_puchased_item_2018 =  get_last_purchased_item(transactions_2018_copy,rfm_segmentation_2018[['LastPurshaceDate']],year=2018)\nlast_puchased_item_2019 =  get_last_purchased_item(transactions_2019_copy,rfm_segmentation_2019[['LastPurshaceDate']],year=2019)\nlast_puchased_item_2020 =  get_last_purchased_item(transactions_2020_copy,rfm_segmentation_2020[['LastPurshaceDate']],year=2020)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:05:40.532297Z","iopub.execute_input":"2022-03-20T07:05:40.532688Z","iopub.status.idle":"2022-03-20T07:06:16.540244Z","shell.execute_reply.started":"2022-03-20T07:05:40.532646Z","shell.execute_reply":"2022-03-20T07:06:16.539192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (3) Get most frequent items per user","metadata":{}},{"cell_type":"code","source":"def get_most_frequent_item(transactions_df,year):\n    temp = transactions_df.groupby(transactions_df.index).agg({\"article_id\":lambda x:Counter(x)})\n    temp = temp['article_id'].apply(lambda x:x.most_common(n=1)[0][0])\n    temp = pd.DataFrame(temp)\n    temp.columns = [f'most_frequent_item_{str(year)}']\n    return temp","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:50.352224Z","iopub.execute_input":"2022-03-20T06:29:50.352497Z","iopub.status.idle":"2022-03-20T06:29:50.361648Z","shell.execute_reply.started":"2022-03-20T06:29:50.352458Z","shell.execute_reply":"2022-03-20T06:29:50.360380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmost_frequent_item_2018 = get_most_frequent_item(transactions_2018_copy,year=2018)\nmost_frequent_item_2019 = get_most_frequent_item(transactions_2019_copy,year=2019)\nmost_frequent_item_2020 = get_most_frequent_item(transactions_2020_copy,year=2020)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:29:50.362943Z","iopub.execute_input":"2022-03-20T06:29:50.363222Z","iopub.status.idle":"2022-03-20T06:30:57.390187Z","shell.execute_reply.started":"2022-03-20T06:29:50.363195Z","shell.execute_reply":"2022-03-20T06:30:57.388785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (4) Get users monthly money spending","metadata":{}},{"cell_type":"code","source":"def get_month_breakdown(df1,df2,df3):\n    df1 = df1.copy()\n    df2 = df2.copy()\n    df3 = df3.copy()\n    \n    df1['month'] = df1['t_dat'].apply(lambda x:x.month)\n    df2['month'] = df2['t_dat'].apply(lambda x:x.month)\n    df3['month'] = df3['t_dat'].apply(lambda x:x.month)\n    \n    df1 = df1.groupby([df1.index,'month']).agg({\"price\":sum})\n    df2 = df2.groupby([df2.index,'month']).agg({\"price\":sum})\n    df3 = df3.groupby([df3.index,'month']).agg({\"price\":sum})\n    \n    \n    df1 = df1.pivot_table(index=df1.index,columns='month',values='price',aggfunc=sum,fill_value=0)\n    df2 = df2.pivot_table(index=df2.index,columns='month',values='price',aggfunc=sum,fill_value=0)\n    df3 = df3.pivot_table(index=df3.index,columns='month',values='price',aggfunc=sum,fill_value=0)\n    \n    \n    df1.index = df1.reset_index()['index'].apply(lambda x:x[0]).values\n    df2.index = df2.reset_index()['index'].apply(lambda x:x[0]).values\n    df3.index = df3.reset_index()['index'].apply(lambda x:x[0]).values\n    \n    df1 = df1.groupby(df1.index).sum()\n    df2 = df2.groupby(df2.index).sum()\n    df3 = df3.groupby(df3.index).sum()\n    \n    df = pd.concat([df1,df2,df3])\n    df = df.groupby(df.index).sum()\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:30:57.392012Z","iopub.execute_input":"2022-03-20T06:30:57.392253Z","iopub.status.idle":"2022-03-20T06:30:57.406068Z","shell.execute_reply.started":"2022-03-20T06:30:57.392228Z","shell.execute_reply":"2022-03-20T06:30:57.404722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntransactions_month_view = get_month_breakdown(transactions_2018_copy,transactions_2019_copy,transactions_2020_copy)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:30:57.407691Z","iopub.execute_input":"2022-03-20T06:30:57.408292Z","iopub.status.idle":"2022-03-20T06:34:30.712998Z","shell.execute_reply.started":"2022-03-20T06:30:57.408259Z","shell.execute_reply":"2022-03-20T06:34:30.711471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (5) Simple EDA","metadata":{}},{"cell_type":"code","source":"N = 100_000\nplt.style.use('fivethirtyeight')\n\ncol_names_l = (\"last_purchesed_item_2018\",\"last_purchesed_item_2019\",\"last_purchesed_item_2020\")\ncol_names_f = (\"most_frequent_item_2018\",\"most_frequent_item_2019\",\"most_frequent_item_2020\")\n\ndfs_l = (last_puchased_item_2018.copy(),last_puchased_item_2019.copy(),last_puchased_item_2020.copy())\ndfs_f = (most_frequent_item_2018.copy(),most_frequent_item_2019.copy(),most_frequent_item_2020.copy())","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:06:21.687981Z","iopub.execute_input":"2022-03-20T07:06:21.688500Z","iopub.status.idle":"2022-03-20T07:06:21.700962Z","shell.execute_reply.started":"2022-03-20T07:06:21.688470Z","shell.execute_reply":"2022-03-20T07:06:21.700180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (a) RFM Features","metadata":{}},{"cell_type":"markdown","source":"**Definitions for RFM**\n\n- RECENCY (R): Days since last purchase\n- FREQUENCY (F): Total number of purchases\n- MONETARY VALUE (M): Total money this customer spent.","metadata":{}},{"cell_type":"code","source":"def get_summary():\n    \n    n_days_2018 = transactions_2018_copy['t_dat'].max() - transactions_2018_copy['t_dat'].min()\n    n_days_2019 = transactions_2019_copy['t_dat'].max() - transactions_2019_copy['t_dat'].min()\n    n_days_2020 = transactions_2020_copy['t_dat'].max() - transactions_2020_copy['t_dat'].min()\n    \n    rfm_info_2018 = rfm_segmentation_2018.describe().to_dict()\n    rfm_info_2019 = rfm_segmentation_2019.describe().to_dict()\n    rfm_info_2020 = rfm_segmentation_2020.describe().to_dict()\n    \n    rfm_info = dict()\n\n    rfm_info['2018'] = dict()\n    rfm_info['2019'] = dict()\n    rfm_info['2020'] = dict()\n\n    rfm_info['2018']['mean_frequency'] = rfm_info_2018['Frequency']['mean']\n    rfm_info['2019']['mean_frequency'] = rfm_info_2019['Frequency']['mean']\n    rfm_info['2020']['mean_frequency'] = rfm_info_2020['Frequency']['mean']\n\n    rfm_info['2018']['mean_monetary'] = rfm_info_2018['Monetary']['mean']\n    rfm_info['2019']['mean_monetary'] = rfm_info_2019['Monetary']['mean']\n    rfm_info['2020']['mean_monetary'] = rfm_info_2020['Monetary']['mean']\n\n    rfm_info['2018']['median_frequency'] =rfm_segmentation_2018['Frequency'].median()\n    rfm_info['2019']['median_frequency'] =rfm_segmentation_2019['Frequency'].median()\n    rfm_info['2020']['median_frequency'] =rfm_segmentation_2020['Frequency'].median()\n\n    rfm_info['2018']['median_monetary'] = rfm_segmentation_2018['Monetary'].median()\n    rfm_info['2019']['median_monetary'] = rfm_segmentation_2019['Monetary'].median()\n    rfm_info['2020']['median_monetary'] = rfm_segmentation_2020['Monetary'].median()\n\n    rfm_info['2018']['total_money_spent'] = rfm_segmentation_2018['Monetary'].sum()\n    rfm_info['2019']['total_money_spent'] = rfm_segmentation_2019['Monetary'].sum()\n    rfm_info['2020']['total_money_spent'] = rfm_segmentation_2020['Monetary'].sum()\n\n    rfm_info['2018']['cutomer_count'] = int(rfm_segmentation_2018.shape[0])\n    rfm_info['2019']['cutomer_count'] = int(rfm_segmentation_2019.shape[0])\n    rfm_info['2020']['cutomer_count'] = int(rfm_segmentation_2020.shape[0])\n    \n    rfm_info['2018']['n_days'] = n_days_2018.days\n    rfm_info['2019']['n_days'] = n_days_2019.days\n    rfm_info['2020']['n_days'] = n_days_2020.days\n    \n    rfm_info['2018']['customers/days'] = rfm_info['2018']['cutomer_count']/n_days_2018.days\n    rfm_info['2019']['customers/days'] = rfm_info['2019']['cutomer_count']/n_days_2019.days\n    rfm_info['2020']['customers/days'] = rfm_info['2020']['cutomer_count']/n_days_2020.days\n    \n    rfm_info['2018']['money/days'] = rfm_info['2018']['total_money_spent']/n_days_2018.days\n    rfm_info['2019']['money/days'] = rfm_info['2019']['total_money_spent']/n_days_2019.days\n    rfm_info['2020']['money/days'] = rfm_info['2020']['total_money_spent']/n_days_2020.days    \n    \n    return pd.DataFrame(rfm_info).T","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:34:30.722862Z","iopub.execute_input":"2022-03-20T06:34:30.723183Z","iopub.status.idle":"2022-03-20T06:34:30.744638Z","shell.execute_reply.started":"2022-03-20T06:34:30.723145Z","shell.execute_reply":"2022-03-20T06:34:30.743820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info = get_summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:34:30.745993Z","iopub.execute_input":"2022-03-20T06:34:30.747462Z","iopub.status.idle":"2022-03-20T06:34:31.422549Z","shell.execute_reply.started":"2022-03-20T06:34:30.747424Z","shell.execute_reply":"2022-03-20T06:34:31.421663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info.style.background_gradient(cmap='viridis',axis='rows')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:54:02.719171Z","iopub.execute_input":"2022-03-20T06:54:02.719408Z","iopub.status.idle":"2022-03-20T06:54:02.740597Z","shell.execute_reply.started":"2022-03-20T06:54:02.719385Z","shell.execute_reply":"2022-03-20T06:54:02.739408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (b) Last purchesed items","metadata":{}},{"cell_type":"code","source":"articles = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T06:43:29.779472Z","iopub.execute_input":"2022-03-20T06:43:29.779852Z","iopub.status.idle":"2022-03-20T06:43:30.680157Z","shell.execute_reply.started":"2022-03-20T06:43:29.779822Z","shell.execute_reply":"2022-03-20T06:43:30.679515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_product_popularity(product_df,col_names,dfs,product_feature='prod_name'):\n    \n    product_df = product_df.copy()\n    \n    col_name1,col_name2,col_name3 = col_names\n    df_2018,df_2019,df_2020 = dfs\n    \n    mapper = dict(zip(product_df['article_id'].values,product_df[product_feature].values))\n    \n    df1 = pd.DataFrame(index=df_2018.index,columns=[col_name1])\n    df2 = pd.DataFrame(index=df_2019.index,columns=[col_name2])\n    df3 = pd.DataFrame(index=df_2020.index,columns=[col_name3])\n    \n    df1[col_name1] = df_2018[col_name1].map(mapper)\n    df2[col_name2] = df_2019[col_name2].map(mapper)\n    df3[col_name3] = df_2020[col_name3].map(mapper)\n    \n    _,axi = plt.subplots(1,3,figsize=(15,9))\n\n    ax = df1[col_name1].value_counts().sort_values().tail(20).plot.barh(ax=axi[0],title='2018')\n    ax = df2[col_name2].value_counts().sort_values().tail(20).plot.barh(ax=axi[1],title='2019')\n    ax = df3[col_name3].value_counts().sort_values().tail(20).plot.barh(ax=axi[2],title='2020')\n\n    plt.tight_layout()\n    plt.grid(False)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:12:14.397739Z","iopub.execute_input":"2022-03-20T07:12:14.397979Z","iopub.status.idle":"2022-03-20T07:12:14.407968Z","shell.execute_reply.started":"2022-03-20T07:12:14.397956Z","shell.execute_reply":"2022-03-20T07:12:14.407252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (i) Product name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_l,dfs=dfs_l)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:12:15.188321Z","iopub.execute_input":"2022-03-20T07:12:15.188889Z","iopub.status.idle":"2022-03-20T07:12:16.936838Z","shell.execute_reply.started":"2022-03-20T07:12:15.188857Z","shell.execute_reply":"2022-03-20T07:12:16.935676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (ii) Product type name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_l,dfs=dfs_l,product_feature='product_type_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:12:41.508017Z","iopub.execute_input":"2022-03-20T07:12:41.508405Z","iopub.status.idle":"2022-03-20T07:12:42.770743Z","shell.execute_reply.started":"2022-03-20T07:12:41.508372Z","shell.execute_reply":"2022-03-20T07:12:42.769174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (iii) Index group name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_l,dfs=dfs_l,product_feature='index_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:12:55.333489Z","iopub.execute_input":"2022-03-20T07:12:55.333799Z","iopub.status.idle":"2022-03-20T07:12:56.577874Z","shell.execute_reply.started":"2022-03-20T07:12:55.333767Z","shell.execute_reply":"2022-03-20T07:12:56.575900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (iv) Section name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_l,dfs=dfs_l,product_feature='section_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:13:05.069363Z","iopub.execute_input":"2022-03-20T07:13:05.070135Z","iopub.status.idle":"2022-03-20T07:13:06.757600Z","shell.execute_reply.started":"2022-03-20T07:13:05.070101Z","shell.execute_reply":"2022-03-20T07:13:06.756309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (c) Most frequent items","metadata":{}},{"cell_type":"markdown","source":"#### (i) Product name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_f,dfs=dfs_f)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:14:06.199251Z","iopub.execute_input":"2022-03-20T07:14:06.199551Z","iopub.status.idle":"2022-03-20T07:14:07.703586Z","shell.execute_reply.started":"2022-03-20T07:14:06.199521Z","shell.execute_reply":"2022-03-20T07:14:07.702383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (ii) Product type name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_f,dfs=dfs_f,product_feature='product_type_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:15:46.503647Z","iopub.execute_input":"2022-03-20T07:15:46.504287Z","iopub.status.idle":"2022-03-20T07:15:48.140728Z","shell.execute_reply.started":"2022-03-20T07:15:46.504240Z","shell.execute_reply":"2022-03-20T07:15:48.139471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (iii) Index group name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_f,dfs=dfs_f,product_feature='index_group_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:17:05.837981Z","iopub.execute_input":"2022-03-20T07:17:05.838262Z","iopub.status.idle":"2022-03-20T07:17:06.667536Z","shell.execute_reply.started":"2022-03-20T07:17:05.838237Z","shell.execute_reply":"2022-03-20T07:17:06.666459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### (iv) Section name trend","metadata":{}},{"cell_type":"code","source":"plot_product_popularity(articles,col_names=col_names_f,dfs=dfs_f,product_feature='section_name')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:17:47.653465Z","iopub.execute_input":"2022-03-20T07:17:47.653702Z","iopub.status.idle":"2022-03-20T07:17:48.767835Z","shell.execute_reply.started":"2022-03-20T07:17:47.653678Z","shell.execute_reply":"2022-03-20T07:17:48.766713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (d) Money spending","metadata":{}},{"cell_type":"markdown","source":"### Check sparsity","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,20))\nplt.spy(transactions_month_view.sample(200).T)\nplt.tight_layout()\nplt.axis('off')\nplt.grid(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:20:58.972982Z","iopub.execute_input":"2022-03-20T07:20:58.973462Z","iopub.status.idle":"2022-03-20T07:20:59.272251Z","shell.execute_reply.started":"2022-03-20T07:20:58.973435Z","shell.execute_reply":"2022-03-20T07:20:59.271610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nax = transactions_month_view.sum(axis=0).plot.bar()\nax.set_xticklabels([\"Jan\",\"Feb\",\"Mar\",\"Apr\",\"May\",\"Jun\",\"Jul\",\"Aug\",\"Sep\",\"Oct\",\"Nov\",\"Dec\"])\nax.set_ylabel(\"Total Money Spent\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:26:14.848546Z","iopub.execute_input":"2022-03-20T07:26:14.848789Z","iopub.status.idle":"2022-03-20T07:26:15.087524Z","shell.execute_reply.started":"2022-03-20T07:26:14.848766Z","shell.execute_reply":"2022-03-20T07:26:15.086268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (6) Save Datasets","metadata":{}},{"cell_type":"code","source":"rfm_segmentation_2018.reset_index().to_csv(\"rfm_features_2018.csv\")\nrfm_segmentation_2019.reset_index().to_csv(\"rfm_features_2019.csv\")\nrfm_segmentation_2020.reset_index().to_csv(\"rfm_features_2020.csv\")\n\ntransactions_2018_copy.reset_index().to_csv(\"transactions_2018.csv\")\ntransactions_2019_copy.reset_index().to_csv(\"transactions_2019.csv\")\ntransactions_2020_copy.reset_index().to_csv(\"transactions_2020.csv\")\n\nlast_puchased_item_2018.reset_index().to_csv(\"last_purchased_items_2018.csv\")\nlast_puchased_item_2019.reset_index().to_csv(\"last_purchased_items_2019.csv\")\nlast_puchased_item_2020.reset_index().to_csv(\"last_purchased_items_2020.csv\")\n\nmost_frequent_item_2018.reset_index().to_csv(\"most_frequent_items.csv\")\nmost_frequent_item_2019.reset_index().to_csv(\"most_frequent_items.csv\")\nmost_frequent_item_2020.reset_index().to_csv(\"most_frequent_items.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T07:38:47.544342Z","iopub.execute_input":"2022-03-20T07:38:47.544825Z"},"trusted":true},"execution_count":null,"outputs":[]}]}