{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n### Objective of the Notebook: \nCreate customer aggregates to capture key attributes based on past purchases, which can be used for further EDA or modeling\n\n### References: \n- https://www.kaggle.com/cdeotte/recommend-items-purchased-together-0-021\n- https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/308635\n","metadata":{}},{"cell_type":"markdown","source":"# Data Loading and Memory Reduction","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport cudf\nimport dask_cudf\n\n\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport matplotlib.image as mpimg\nimport scipy\n\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-03T09:55:20.549075Z","iopub.execute_input":"2022-03-03T09:55:20.549320Z","iopub.status.idle":"2022-03-03T09:55:22.972206Z","shell.execute_reply.started":"2022-03-03T09:55:20.549247Z","shell.execute_reply":"2022-03-03T09:55:22.971371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ntransactions['customer_id'] = transactions['customer_id'].str[-16:].str.hex_to_int().astype('int64')\ntransactions['article_id'] = transactions.article_id.astype('int32')\ntransactions.t_dat = cudf.to_datetime(transactions.t_dat)\ntransactions.to_parquet('train.pqt',index=False)\nprint(transactions.shape)\ntransactions['priceK'] = transactions['price'] * 1000\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:22.975606Z","iopub.execute_input":"2022-03-03T09:55:22.975845Z","iopub.status.idle":"2022-03-03T09:55:28.058769Z","shell.execute_reply.started":"2022-03-03T09:55:22.975814Z","shell.execute_reply":"2022-03-03T09:55:28.057987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_id_decile = []\nfor i in range(10):\n    j = (i + 1) * 0.1\n    customer_id_decile.append(transactions.customer_id.quantile(j)) \ncustomer_id_decile   ","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:28.060210Z","iopub.execute_input":"2022-03-03T09:55:28.060685Z","iopub.status.idle":"2022-03-03T09:55:28.496961Z","shell.execute_reply.started":"2022-03-03T09:55:28.060643Z","shell.execute_reply":"2022-03-03T09:55:28.496130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load articles dataset and incorporate into transactions","metadata":{}},{"cell_type":"code","source":"articles = cudf.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\nprint(articles.columns)\narticles.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:28.500336Z","iopub.execute_input":"2022-03-03T09:55:28.500561Z","iopub.status.idle":"2022-03-03T09:55:28.683981Z","shell.execute_reply.started":"2022-03-03T09:55:28.500533Z","shell.execute_reply":"2022-03-03T09:55:28.683100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions = transactions.sample(10000)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:28.685561Z","iopub.execute_input":"2022-03-03T09:55:28.685849Z","iopub.status.idle":"2022-03-03T09:55:28.772098Z","shell.execute_reply.started":"2022-03-03T09:55:28.685811Z","shell.execute_reply":"2022-03-03T09:55:28.771117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########3\n# # dask_cudf merge is not working as expected. reference: https://github.com/rapidsai/cudf/issues/2694 \n#########\n\n# dask_transactions = dask_cudf.from_cudf(transactions,npartitions=20)\n# dask_articles = dask_cudf.from_cudf(articles,npartitions=20)\n# transactionsEnriched = dask_transactions.merge(dask_articles,on='article_id',how='left')\n# transactionsEnriched.compute().head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:28.774136Z","iopub.execute_input":"2022-03-03T09:55:28.774984Z","iopub.status.idle":"2022-03-03T09:55:28.781175Z","shell.execute_reply.started":"2022-03-03T09:55:28.774911Z","shell.execute_reply":"2022-03-03T09:55:28.780434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask as dask\ndask_transactions = dask.dataframe.from_pandas(transactions.to_pandas(),npartitions=20)\ndask_articles = dask.dataframe.from_pandas(articles.to_pandas(),npartitions=20)\ntransactionsEnriched = dask_transactions.merge(dask_articles,on='article_id',how='left')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:28.782771Z","iopub.execute_input":"2022-03-03T09:55:28.783376Z","iopub.status.idle":"2022-03-03T09:55:31.249399Z","shell.execute_reply.started":"2022-03-03T09:55:28.783335Z","shell.execute_reply":"2022-03-03T09:55:31.248507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# try:\n#     shutil.rmtree('df_transactionsEnriched.pqt')\n# except:\n#     print('na')\n# transactionsEnriched.to_parquet('df_transactionsEnriched.pqt')\n# transactionsEnriched.to_csv('df_transactionsEnriched.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.250956Z","iopub.execute_input":"2022-03-03T09:55:31.251233Z","iopub.status.idle":"2022-03-03T09:55:31.255317Z","shell.execute_reply.started":"2022-03-03T09:55:31.251194Z","shell.execute_reply":"2022-03-03T09:55:31.254563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_transactionsEnriched = pd.DataFrame(transactionsEnriched)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.257046Z","iopub.execute_input":"2022-03-03T09:55:31.257450Z","iopub.status.idle":"2022-03-03T09:55:31.264742Z","shell.execute_reply.started":"2022-03-03T09:55:31.257409Z","shell.execute_reply":"2022-03-03T09:55:31.263919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_transactionsEnriched = pd.read_parquet('df_transactionsEnriched.pqt')\n# # df_transactionsEnriched = pd.read_csv('df_transactionsEnriched.csv')\n# df_transactionsEnriched.head()\n\n# Too big to be read as regular pandas","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.266126Z","iopub.execute_input":"2022-03-03T09:55:31.266523Z","iopub.status.idle":"2022-03-03T09:55:31.273804Z","shell.execute_reply.started":"2022-03-03T09:55:31.266480Z","shell.execute_reply":"2022-03-03T09:55:31.273069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactionsEnrichedDict = {}\nfor i in range(10):\n    upper = customer_id_decile[i]\n    if i == 0:\n        transactionsEnrichedDict[i] = transactionsEnriched[(transactionsEnriched.customer_id <= upper)]\n    else:        \n        lower = customer_id_decile[i-1]\n        transactionsEnrichedDict[i] = transactionsEnriched[(transactionsEnriched.customer_id > lower) &  (transactionsEnriched.customer_id <= upper)]","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.275150Z","iopub.execute_input":"2022-03-03T09:55:31.275603Z","iopub.status.idle":"2022-03-03T09:55:31.305353Z","shell.execute_reply.started":"2022-03-03T09:55:31.275564Z","shell.execute_reply":"2022-03-03T09:55:31.304529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_cols = [\n       'product_type_name', 'product_group_name', 'graphical_appearance_no',\n       'graphical_appearance_name', 'colour_group_name',\n       'perceived_colour_value_name',\n       'perceived_colour_master_name',\n       'department_name', 'index_name',\n       'index_group_name', 'section_name',\n       'garment_group_name']\n\ndictAggr = {'article_id':'count',\n            'priceK':['mean','sum']\n           }\n\nfor col in category_cols:\n    dictAggr[col]= pd.Series.mode\n\nprint(dictAggr)\n    \ndef countItem(group, columnCat, columnCat_item):\n    dfOutput = group[group[columnCat]==columnCat_item]['customer_id'].count()\n    return dfOutput\n\ndef countItemDict(group, columnCat, columnCat_itemlist):\n    Output = {}\n    for columnCat_item in columnCat_itemlist:\n        Output[columnCat_item] = group[group[columnCat]==columnCat_item]['customer_id'].count()\n    return Output\n\n","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.307969Z","iopub.execute_input":"2022-03-03T09:55:31.308179Z","iopub.status.idle":"2022-03-03T09:55:31.319469Z","shell.execute_reply.started":"2022-03-03T09:55:31.308155Z","shell.execute_reply":"2022-03-03T09:55:31.318486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactionsSample = transactionsEnrichedDict[0]\ndict_CustomerArticleAttributes = {}\ndict_df_NumPurch_index_group_name = {}\ndict_df_NumPurch_garment_group_name = {}\n\nfor i in range(10):\n    \n    df_transactionsEnriched = transactionsEnrichedDict[i].compute()\n    \n    # Aggregate for most frequent item \n    CustomerArticleAttributes = df_transactionsEnriched.groupby('customer_id').agg(dictAggr).reset_index()\n    CustomerArticleAttributes.columns = [' '.join(col).strip() for col in CustomerArticleAttributes.columns.values]\n    CustomerArticleAttributes.rename(columns={'article_id count':'count'},inplace=True)\n    for col in category_cols:\n        CustomerArticleAttributes.rename(columns={col+' mode':'mostfreq_'+col},inplace=True)\n        \n    # Count of specific 'index_group_name' category\n    columnCat_itemlist = df_transactionsEnriched['index_group_name'].unique()\n    columnCat = 'index_group_name'\n    df_NumPurch_index_group_name = df_transactionsEnriched.groupby(['customer_id']).apply(lambda grp: countItemDict(grp,columnCat,columnCat_itemlist)).reset_index()\n    df_NumPurch_index_group_name.columns = ['customer_id','numpurch_dict']    \n    for item in columnCat_itemlist:\n        colname = 'numpurchased_'+item\n        df_NumPurch_index_group_name[colname] = [row[item] for row in df_NumPurch_index_group_name.numpurch_dict]\n    \n    # Count of specific 'garment_group_name' category\n    # Get Top 15 group names\n    df_transactionsEnriched.groupby(['garment_group_name'])['customer_id'].agg('count').sort_values(ascending=False).head(15).index\n    columnCat = 'garment_group_name'\n    columnCat_itemlist = df_transactionsEnriched.groupby(['garment_group_name'])['customer_id'].agg('count').sort_values(ascending=False).head(15).index\n    df_NumPurch_garment_group_name = df_transactionsEnriched.groupby(['customer_id']).apply(lambda grp: countItemDict(grp,columnCat,columnCat_itemlist)).reset_index()\n    df_NumPurch_garment_group_name.columns = ['customer_id','numpurch_dict']\n    for item in columnCat_itemlist:\n        colname = 'numpurchased_'+item\n        df_NumPurch_garment_group_name[colname] = [row[item] for row in df_NumPurch_garment_group_name.numpurch_dict]\n        \n    dict_CustomerArticleAttributes[i] = CustomerArticleAttributes\n    dict_df_NumPurch_index_group_name[i] = df_NumPurch_index_group_name\n    dict_df_NumPurch_garment_group_name[i] = df_NumPurch_garment_group_name\n","metadata":{"execution":{"iopub.status.busy":"2022-03-03T09:55:31.324398Z","iopub.execute_input":"2022-03-03T09:55:31.324596Z","iopub.status.idle":"2022-03-03T14:49:53.643410Z","shell.execute_reply.started":"2022-03-03T09:55:31.324572Z","shell.execute_reply":"2022-03-03T14:49:53.640376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_df_NumPurch_index_group_name[1].head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:49:53.650033Z","iopub.execute_input":"2022-03-03T14:49:53.650706Z","iopub.status.idle":"2022-03-03T14:49:53.670112Z","shell.execute_reply.started":"2022-03-03T14:49:53.650662Z","shell.execute_reply":"2022-03-03T14:49:53.669295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Combine across chunks","metadata":{}},{"cell_type":"code","source":"for i in range(10):\n    if i == 0:\n        CustomerArticleAttributes = dict_CustomerArticleAttributes[i]\n        df_NumPurch_index_group_name = dict_df_NumPurch_index_group_name[i]\n        df_NumPurch_garment_group_name = dict_df_NumPurch_index_group_name[i]\n    else:\n        CustomerArticleAttributes.append(dict_CustomerArticleAttributes[i])\n        df_NumPurch_index_group_name.append(dict_df_NumPurch_index_group_name[i])\n        df_NumPurch_garment_group_name.append(dict_df_NumPurch_index_group_name[i])\nCustomerArticleAttributes.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:49:53.671541Z","iopub.execute_input":"2022-03-03T14:49:53.671876Z","iopub.status.idle":"2022-03-03T14:49:54.712341Z","shell.execute_reply.started":"2022-03-03T14:49:53.671837Z","shell.execute_reply":"2022-03-03T14:49:54.711575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge into single customer table","metadata":{}},{"cell_type":"code","source":"df_NumPurch_garment_group_name=df_NumPurch_garment_group_name.drop('numpurch_dict',axis=1)\ndf_NumPurch_index_group_name=df_NumPurch_index_group_name.drop('numpurch_dict',axis=1)\nCustomerArticleAttributes.rename(columns={'count':'totalpurchase'},inplace=True)\n\ncustomers = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')\ncustomers['customer_id'] = customers['customer_id'].str[-16:].str.hex_to_int().astype('int64')\ncustomers.head()\n\ncol_objects = CustomerArticleAttributes.columns[4:]\nfor col in col_objects:\n    CustomerArticleAttributes[col] = CustomerArticleAttributes[col].astype(str)\n    \ncudf_NumPurch_garment_group_name = cudf.DataFrame(df_NumPurch_garment_group_name)\ncudf_NumPurch_index_group_name = cudf.DataFrame(df_NumPurch_index_group_name)\ncudf_CustomerArticleAttributes = cudf.DataFrame(CustomerArticleAttributes)\n\ncustomersEnriched = (customers.merge(cudf_NumPurch_garment_group_name,on='customer_id',how='left')\n                     .merge(cudf_NumPurch_index_group_name,on='customer_id',how='left')\n                     .merge(cudf_CustomerArticleAttributes,on='customer_id',how='left')\n                    )\ncustomersEnriched.to_csv('customersEnriched.csv')\n# os.remove('./customersEnriched.pqt')\n# customersEnriched.to_parquet('customersEnriched.pqt')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:49:54.713598Z","iopub.execute_input":"2022-03-03T14:49:54.714114Z","iopub.status.idle":"2022-03-03T14:50:05.065800Z","shell.execute_reply.started":"2022-03-03T14:49:54.714073Z","shell.execute_reply":"2022-03-03T14:50:05.064907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# #####################################\n# Previous Set of Codes; not deleted for reference of workflow before combining them\n# #####################################","metadata":{}},{"cell_type":"markdown","source":"# Create Customer Attributes based on Purchased Products","metadata":{}},{"cell_type":"code","source":"# category_cols = [\n#        'product_type_name', 'product_group_name', 'graphical_appearance_no',\n#        'graphical_appearance_name', 'colour_group_name',\n#        'perceived_colour_value_name',\n#        'perceived_colour_master_name',\n#        'department_name', 'index_name',\n#        'index_group_name', 'section_name',\n#        'garment_group_name']\n\n# dictAggr = {'article_id':'count',\n#             'priceK':['mean','sum']\n#            }\n\n# for col in category_cols:\n#     dictAggr[col]= pd.Series.mode\n\n# dictAggr","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.067658Z","iopub.execute_input":"2022-03-03T14:50:05.068173Z","iopub.status.idle":"2022-03-03T14:50:05.072587Z","shell.execute_reply.started":"2022-03-03T14:50:05.068132Z","shell.execute_reply":"2022-03-03T14:50:05.071900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CustomerArticleAttributes = df_transactionsEnriched.groupby('customer_id').agg(dictAggr).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.074186Z","iopub.execute_input":"2022-03-03T14:50:05.074869Z","iopub.status.idle":"2022-03-03T14:50:05.081763Z","shell.execute_reply.started":"2022-03-03T14:50:05.074810Z","shell.execute_reply":"2022-03-03T14:50:05.081101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CustomerArticleAttributes.columns = [' '.join(col).strip() for col in CustomerArticleAttributes.columns.values]\n# CustomerArticleAttributes.rename(columns={'article_id count':'count'},inplace=True)\n# for col in category_cols:\n#     CustomerArticleAttributes.rename(columns={col+' mode':'mostfreq_'+col},inplace=True)\n# CustomerArticleAttributes.head()\n# # CustomerArticleAttributes.sort_values(('article_id','count'),ascending=False).head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.083418Z","iopub.execute_input":"2022-03-03T14:50:05.084024Z","iopub.status.idle":"2022-03-03T14:50:05.090810Z","shell.execute_reply.started":"2022-03-03T14:50:05.083985Z","shell.execute_reply":"2022-03-03T14:50:05.090145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Purchase of Specific Category Flags","metadata":{}},{"cell_type":"markdown","source":"### Create and test the function","metadata":{}},{"cell_type":"code","source":"# # Do one example\n# columnCat = 'index_group_name'\n# columnCat_item = 'Ladieswear'\n# columnCat_itemlist = ['Ladieswear','Menswear']\n\n# def countItem(group, columnCat, columnCat_item):\n#     dfOutput = group[group[columnCat]==columnCat_item]['customer_id'].count()\n#     return dfOutput\n# def countItemDict(group, columnCat, columnCat_itemlist):\n#     Output = {}\n#     for columnCat_item in columnCat_itemlist:\n#         Output[columnCat_item] = group[group[columnCat]==columnCat_item]['customer_id'].count()\n#     return Output","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.091903Z","iopub.execute_input":"2022-03-03T14:50:05.092226Z","iopub.status.idle":"2022-03-03T14:50:05.102191Z","shell.execute_reply.started":"2022-03-03T14:50:05.092190Z","shell.execute_reply":"2022-03-03T14:50:05.101414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dfColAggr = df_transactionsEnriched.groupby(['customer_id']).apply(lambda grp: countItemDict(grp,columnCat,columnCat_itemlist)).reset_index()\n# dfColAggr.columns=['customer_id','index_group_name_dict']\n# dfColAggr.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.103438Z","iopub.execute_input":"2022-03-03T14:50:05.104410Z","iopub.status.idle":"2022-03-03T14:50:05.110628Z","shell.execute_reply.started":"2022-03-03T14:50:05.104370Z","shell.execute_reply":"2022-03-03T14:50:05.109960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create for index_group_name","metadata":{}},{"cell_type":"code","source":"# df_transactionsEnriched.groupby('index_group_name')['customer_id'].agg('count').sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.112179Z","iopub.execute_input":"2022-03-03T14:50:05.112799Z","iopub.status.idle":"2022-03-03T14:50:05.119326Z","shell.execute_reply.started":"2022-03-03T14:50:05.112760Z","shell.execute_reply":"2022-03-03T14:50:05.118577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columnCat_itemlist = df_transactionsEnriched['index_group_name'].unique()\n# columnCat = 'index_group_name'\n# df_NumPurch_index_group_name = df_transactionsEnriched.groupby(['customer_id']).apply(lambda grp: countItemDict(grp,columnCat,columnCat_itemlist)).reset_index()\n# df_NumPurch_index_group_name.columns = ['customer_id','numpurch_dict']\n# df_NumPurch_index_group_name.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.120621Z","iopub.execute_input":"2022-03-03T14:50:05.121586Z","iopub.status.idle":"2022-03-03T14:50:05.128425Z","shell.execute_reply.started":"2022-03-03T14:50:05.121548Z","shell.execute_reply":"2022-03-03T14:50:05.127587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for item in columnCat_itemlist:\n#     colname = 'numpurchased_'+item\n#     df_NumPurch_index_group_name[colname] = [row[item] for row in df_NumPurch_index_group_name.numpurch_dict]\n# df_NumPurch_index_group_name.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.129816Z","iopub.execute_input":"2022-03-03T14:50:05.130577Z","iopub.status.idle":"2022-03-03T14:50:05.137457Z","shell.execute_reply.started":"2022-03-03T14:50:05.130537Z","shell.execute_reply":"2022-03-03T14:50:05.136734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create for garment_group_name","metadata":{}},{"cell_type":"code","source":"# df_transactionsEnriched.groupby(['garment_group_name','index_group_name'])['customer_id'].agg('count').head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.138945Z","iopub.execute_input":"2022-03-03T14:50:05.139373Z","iopub.status.idle":"2022-03-03T14:50:05.145998Z","shell.execute_reply.started":"2022-03-03T14:50:05.139335Z","shell.execute_reply":"2022-03-03T14:50:05.145226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_transactionsEnriched.groupby(['garment_group_name'])['customer_id'].agg('count').sort_values(ascending=False).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.147279Z","iopub.execute_input":"2022-03-03T14:50:05.148239Z","iopub.status.idle":"2022-03-03T14:50:05.157431Z","shell.execute_reply.started":"2022-03-03T14:50:05.148201Z","shell.execute_reply":"2022-03-03T14:50:05.156763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Get Top 15 group names\n# df_transactionsEnriched.groupby(['garment_group_name'])['customer_id'].agg('count').sort_values(ascending=False).head(15).index","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.158924Z","iopub.execute_input":"2022-03-03T14:50:05.160446Z","iopub.status.idle":"2022-03-03T14:50:05.166290Z","shell.execute_reply.started":"2022-03-03T14:50:05.160407Z","shell.execute_reply":"2022-03-03T14:50:05.165561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columnCat = 'garment_group_name'\n# columnCat_itemlist = df_transactionsEnriched.groupby(['garment_group_name'])['customer_id'].agg('count').sort_values(ascending=False).head(15).index\n# df_NumPurch_garment_group_name = df_transactionsEnriched.groupby(['customer_id']).apply(lambda grp: countItemDict(grp,columnCat,columnCat_itemlist)).reset_index()\n# df_NumPurch_garment_group_name.columns = ['customer_id','numpurch_dict']\n# df_NumPurch_garment_group_name.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.168397Z","iopub.execute_input":"2022-03-03T14:50:05.169067Z","iopub.status.idle":"2022-03-03T14:50:05.174426Z","shell.execute_reply.started":"2022-03-03T14:50:05.169028Z","shell.execute_reply":"2022-03-03T14:50:05.173708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for item in columnCat_itemlist:\n#     colname = 'numpurchased_'+item\n#     df_NumPurch_garment_group_name[colname] = [row[item] for row in df_NumPurch_garment_group_name.numpurch_dict]\n# df_NumPurch_garment_group_name.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.176090Z","iopub.execute_input":"2022-03-03T14:50:05.176357Z","iopub.status.idle":"2022-03-03T14:50:05.182757Z","shell.execute_reply.started":"2022-03-03T14:50:05.176324Z","shell.execute_reply":"2022-03-03T14:50:05.182050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_NumPurch_garment_group_name.sort_values('numpurchased_Jersey Fancy',ascending=False).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.185063Z","iopub.execute_input":"2022-03-03T14:50:05.185837Z","iopub.status.idle":"2022-03-03T14:50:05.191129Z","shell.execute_reply.started":"2022-03-03T14:50:05.185800Z","shell.execute_reply":"2022-03-03T14:50:05.190374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge into original customer attributes","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Recap of the tables to be merged\n- df_NumPurch_garment_group_name\n- df_NumPurch_index_group_name\n- CustomerArticleAttributes","metadata":{}},{"cell_type":"code","source":"# df_NumPurch_garment_group_name=df_NumPurch_garment_group_name.drop('numpurch_dict',axis=1)\n# df_NumPurch_index_group_name=df_NumPurch_index_group_name.drop('numpurch_dict',axis=1)\n# CustomerArticleAttributes.rename(columns={'count':'totalpurchase'},inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.192812Z","iopub.execute_input":"2022-03-03T14:50:05.193339Z","iopub.status.idle":"2022-03-03T14:50:05.198973Z","shell.execute_reply.started":"2022-03-03T14:50:05.193300Z","shell.execute_reply":"2022-03-03T14:50:05.198253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_NumPurch_garment_group_name=df_NumPurch_garment_group_name.drop('numpurch_dict',axis=1)\n# df_NumPurch_garment_group_name.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.200472Z","iopub.execute_input":"2022-03-03T14:50:05.201353Z","iopub.status.idle":"2022-03-03T14:50:05.206453Z","shell.execute_reply.started":"2022-03-03T14:50:05.201314Z","shell.execute_reply":"2022-03-03T14:50:05.205768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_NumPurch_index_group_name=df_NumPurch_index_group_name.drop('numpurch_dict',axis=1)\n# df_NumPurch_index_group_name.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.207652Z","iopub.execute_input":"2022-03-03T14:50:05.208591Z","iopub.status.idle":"2022-03-03T14:50:05.214180Z","shell.execute_reply.started":"2022-03-03T14:50:05.208532Z","shell.execute_reply":"2022-03-03T14:50:05.213301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CustomerArticleAttributes.rename(columns={'count':'totalpurchase'},inplace=True)\n# CustomerArticleAttributes.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.215674Z","iopub.execute_input":"2022-03-03T14:50:05.216291Z","iopub.status.idle":"2022-03-03T14:50:05.221440Z","shell.execute_reply.started":"2022-03-03T14:50:05.216252Z","shell.execute_reply":"2022-03-03T14:50:05.220693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# customers = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')\n# customers['customer_id'] = customers['customer_id'].str[-16:].str.hex_to_int().astype('int64')\n# customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.222668Z","iopub.execute_input":"2022-03-03T14:50:05.223473Z","iopub.status.idle":"2022-03-03T14:50:05.229289Z","shell.execute_reply.started":"2022-03-03T14:50:05.223437Z","shell.execute_reply":"2022-03-03T14:50:05.228405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(CustomerArticleAttributes.columns)\n# print(CustomerArticleAttributes.columns[4:])\n\n# col_objects = CustomerArticleAttributes.columns[4:]\n# for col in col_objects:\n#     CustomerArticleAttributes[col] = CustomerArticleAttributes[col].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.230496Z","iopub.execute_input":"2022-03-03T14:50:05.231327Z","iopub.status.idle":"2022-03-03T14:50:05.238249Z","shell.execute_reply.started":"2022-03-03T14:50:05.231288Z","shell.execute_reply":"2022-03-03T14:50:05.237574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cudf_NumPurch_garment_group_name = cudf.DataFrame(df_NumPurch_garment_group_name)\n# cudf_NumPurch_index_group_name = cudf.DataFrame(df_NumPurch_index_group_name)\n# cudf_CustomerArticleAttributes = cudf.DataFrame(CustomerArticleAttributes)","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.239504Z","iopub.execute_input":"2022-03-03T14:50:05.239879Z","iopub.status.idle":"2022-03-03T14:50:05.246299Z","shell.execute_reply.started":"2022-03-03T14:50:05.239840Z","shell.execute_reply":"2022-03-03T14:50:05.245571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# customersEnriched = (customers.merge(cudf_NumPurch_garment_group_name,on='customer_id',how='left')\n#                      .merge(cudf_NumPurch_index_group_name,on='customer_id',how='left')\n#                      .merge(cudf_CustomerArticleAttributes,on='customer_id',how='left')\n#                     )\n# customersEnriched[customersEnriched.numpurchased_Shoes>0].head()","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.247845Z","iopub.execute_input":"2022-03-03T14:50:05.248131Z","iopub.status.idle":"2022-03-03T14:50:05.254574Z","shell.execute_reply.started":"2022-03-03T14:50:05.248093Z","shell.execute_reply":"2022-03-03T14:50:05.253840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# customersEnriched.to_csv('customersEnriched.csv')\n# os.remove('./customersEnriched.pqt')\n# customersEnriched.to_parquet('customersEnriched.pqt')","metadata":{"execution":{"iopub.status.busy":"2022-03-03T14:50:05.256141Z","iopub.execute_input":"2022-03-03T14:50:05.256418Z","iopub.status.idle":"2022-03-03T14:50:05.263144Z","shell.execute_reply.started":"2022-03-03T14:50:05.256384Z","shell.execute_reply":"2022-03-03T14:50:05.262437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save as csv and parquet","metadata":{}}]}