{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-14T14:01:14.106397Z","iopub.execute_input":"2022-04-14T14:01:14.106669Z","iopub.status.idle":"2022-04-14T14:01:14.113606Z","shell.execute_reply.started":"2022-04-14T14:01:14.106641Z","shell.execute_reply":"2022-04-14T14:01:14.112987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This notebook is used to process the data of H&M RS","metadata":{}},{"cell_type":"markdown","source":"## 1.clean data in article.csv","metadata":{}},{"cell_type":"code","source":"#read article.csv file\narticle = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv')\nprint(article.shape)\narticle.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.114915Z","iopub.execute_input":"2022-04-14T14:01:14.115781Z","iopub.status.idle":"2022-04-14T14:01:14.794279Z","shell.execute_reply.started":"2022-04-14T14:01:14.115744Z","shell.execute_reply":"2022-04-14T14:01:14.793557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## remove name feature only left code feature","metadata":{}},{"cell_type":"code","source":"article.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.795769Z","iopub.execute_input":"2022-04-14T14:01:14.795971Z","iopub.status.idle":"2022-04-14T14:01:14.801449Z","shell.execute_reply.started":"2022-04-14T14:01:14.795946Z","shell.execute_reply":"2022-04-14T14:01:14.800504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_feature = ['article_id', 'product_code', 'product_type_no',\n                'product_group_name', 'graphical_appearance_no',\n                'colour_group_code', 'perceived_colour_value_id', \n                'perceived_colour_master_id', \n                'department_no', 'index_code',\n                'index_group_no', 'section_no',\n                'garment_group_no']\ntext_des = ['detail_desc']\n\n# change string into code\nfrom sklearn.preprocessing import LabelEncoder\nchange_column=['product_group_name', 'index_code']\nfor column in change_column:\n    lbe = LabelEncoder()\n    lbe.fit(article[column].unique())\n    article[column] = lbe.transform(article[column])\n\n#for nn.embedding\narticle_code = article[code_feature]\narticle_code.head()\n\n#for BERT embed\narticle_describe = article[text_des]","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.802511Z","iopub.execute_input":"2022-04-14T14:01:14.802700Z","iopub.status.idle":"2022-04-14T14:01:14.891836Z","shell.execute_reply.started":"2022-04-14T14:01:14.802676Z","shell.execute_reply":"2022-04-14T14:01:14.890813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_code.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.893662Z","iopub.execute_input":"2022-04-14T14:01:14.893902Z","iopub.status.idle":"2022-04-14T14:01:14.902487Z","shell.execute_reply.started":"2022-04-14T14:01:14.893873Z","shell.execute_reply":"2022-04-14T14:01:14.901537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_code.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.903590Z","iopub.execute_input":"2022-04-14T14:01:14.903802Z","iopub.status.idle":"2022-04-14T14:01:14.918895Z","shell.execute_reply.started":"2022-04-14T14:01:14.903777Z","shell.execute_reply":"2022-04-14T14:01:14.917927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_describe.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.919797Z","iopub.execute_input":"2022-04-14T14:01:14.920770Z","iopub.status.idle":"2022-04-14T14:01:14.934279Z","shell.execute_reply.started":"2022-04-14T14:01:14.920722Z","shell.execute_reply":"2022-04-14T14:01:14.933280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_map(data,col_name):\n    key = data[col_name].unique()\n    m = dict(zip(key, range(len(key))))\n    data[col_name] = data[col_name].map(lambda x: m[x])\n    return key, m\n\n_, article_map = build_map(article_code,'article_id')\n_, tmp = build_map(article_code,'product_type_no')","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:14.935210Z","iopub.execute_input":"2022-04-14T14:01:14.935912Z","iopub.status.idle":"2022-04-14T14:01:15.106259Z","shell.execute_reply.started":"2022-04-14T14:01:14.935881Z","shell.execute_reply":"2022-04-14T14:01:15.105383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nif not os.path.exists('process_0'):\n    os.mkdir('process_0')\npd.to_pickle(article_code,'process_0/article_code.pkl')\npd.to_pickle(article_describe,'process_0/article_describe.pkl')\nwith open('process_0/atricle_map.pkl','wb') as f:\n    pickle.dump(article_map,f)\n\n''' # read article_map\nwith open(''process_0/atricle_map.pkl','rb') as f:\n    article_map = pickle.load(f)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:15.107515Z","iopub.execute_input":"2022-04-14T14:01:15.107983Z","iopub.status.idle":"2022-04-14T14:01:15.447263Z","shell.execute_reply.started":"2022-04-14T14:01:15.107938Z","shell.execute_reply":"2022-04-14T14:01:15.446341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## article.csv process done!","metadata":{}},{"cell_type":"markdown","source":"## 2.clean data in customers.csv","metadata":{}},{"cell_type":"code","source":"# read customers.csv\ncustomers = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')\nprint('customers shape:',customers.shape)\ncustomers.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:15.448961Z","iopub.execute_input":"2022-04-14T14:01:15.449270Z","iopub.status.idle":"2022-04-14T14:01:18.929243Z","shell.execute_reply.started":"2022-04-14T14:01:15.449227Z","shell.execute_reply":"2022-04-14T14:01:18.928324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.isnull().sum()\n#there are much missing values in customers.csv, but some can be filled like 'club_member_status'","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:18.933044Z","iopub.execute_input":"2022-04-14T14:01:18.933381Z","iopub.status.idle":"2022-04-14T14:01:19.212356Z","shell.execute_reply.started":"2022-04-14T14:01:18.933335Z","shell.execute_reply":"2022-04-14T14:01:19.211509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.describe()\n#FN and Active seems only have 1.0 value, may be we can fill 0 into missing value.","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:19.213962Z","iopub.execute_input":"2022-04-14T14:01:19.214258Z","iopub.status.idle":"2022-04-14T14:01:19.404340Z","shell.execute_reply.started":"2022-04-14T14:01:19.214219Z","shell.execute_reply":"2022-04-14T14:01:19.403610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.fashion_news_frequency.unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:19.405548Z","iopub.execute_input":"2022-04-14T14:01:19.405778Z","iopub.status.idle":"2022-04-14T14:01:19.486356Z","shell.execute_reply.started":"2022-04-14T14:01:19.405751Z","shell.execute_reply":"2022-04-14T14:01:19.485432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status.unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:19.487246Z","iopub.execute_input":"2022-04-14T14:01:19.487493Z","iopub.status.idle":"2022-04-14T14:01:19.572265Z","shell.execute_reply.started":"2022-04-14T14:01:19.487462Z","shell.execute_reply":"2022-04-14T14:01:19.571120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_column_name = ['customer_id', 'FN', 'Active', 'club_member_status',\n                        'fashion_news_frequency', 'age', 'postal_code']\n\n#fill all missing value\ncustomers.club_member_status[customers.club_member_status.isnull()] = 'PRE-CREATE'\ncustomers.fashion_news_frequency[customers.fashion_news_frequency.isnull()] = 'NONE'\ncustomers.fashion_news_frequency[customers.fashion_news_frequency =='None'] = 'NONE'\ncustomers.FN[customers.FN.isnull()] = 0.\ncustomers.Active[customers.Active.isnull()] = 0.\ncustomers.age[customers.age.isnull()] = customers.age.mode().values[0]","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:19.573752Z","iopub.execute_input":"2022-04-14T14:01:19.574362Z","iopub.status.idle":"2022-04-14T14:01:19.867088Z","shell.execute_reply.started":"2022-04-14T14:01:19.574294Z","shell.execute_reply":"2022-04-14T14:01:19.866269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlbe = LabelEncoder()\nlbe.fit(customers.postal_code)\ncustomers.postal_code = lbe.transform(customers.postal_code)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:19.868380Z","iopub.execute_input":"2022-04-14T14:01:19.868685Z","iopub.status.idle":"2022-04-14T14:01:22.297377Z","shell.execute_reply.started":"2022-04-14T14:01:19.868644Z","shell.execute_reply":"2022-04-14T14:01:22.296747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:22.298634Z","iopub.execute_input":"2022-04-14T14:01:22.299035Z","iopub.status.idle":"2022-04-14T14:01:22.311539Z","shell.execute_reply.started":"2022-04-14T14:01:22.299005Z","shell.execute_reply":"2022-04-14T14:01:22.310717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, customer_map = build_map(customers, 'customer_id')","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:22.313599Z","iopub.execute_input":"2022-04-14T14:01:22.313917Z","iopub.status.idle":"2022-04-14T14:01:24.075256Z","shell.execute_reply.started":"2022-04-14T14:01:22.313877Z","shell.execute_reply":"2022-04-14T14:01:24.074382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('process_0'):\n    os.mkdir('process_0')\npd.to_pickle(customers,'process_0/customers.pkl')\nwith open('process_0/customer_map.pkl','wb') as f:\n    pickle.dump(customer_map,f)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:24.076665Z","iopub.execute_input":"2022-04-14T14:01:24.077222Z","iopub.status.idle":"2022-04-14T14:01:25.028075Z","shell.execute_reply.started":"2022-04-14T14:01:24.077171Z","shell.execute_reply":"2022-04-14T14:01:25.027167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## customers.csv Done!","metadata":{}},{"cell_type":"markdown","source":"## 3.clean data of transactions_train.csv","metadata":{}},{"cell_type":"code","source":"# read transactions_train.csv\ntransactions = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\nprint('transactions shape:',transactions.shape)\ntransactions['customer_id'] = transactions['customer_id'].map(lambda x: customer_map[x])\ntransactions['article_id'] = transactions['article_id'].map(lambda x: article_map[x])\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:01:25.029592Z","iopub.execute_input":"2022-04-14T14:01:25.029886Z","iopub.status.idle":"2022-04-14T14:03:03.744911Z","shell.execute_reply.started":"2022-04-14T14:01:25.029844Z","shell.execute_reply":"2022-04-14T14:03:03.742958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('process_0'):\n    os.mkdir('process_0')\npd.to_pickle(transactions,'process_0/transactions.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:03.746221Z","iopub.execute_input":"2022-04-14T14:03:03.746538Z","iopub.status.idle":"2022-04-14T14:03:06.822957Z","shell.execute_reply.started":"2022-04-14T14:03:03.746495Z","shell.execute_reply":"2022-04-14T14:03:06.822362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('process_0')","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:06.824537Z","iopub.execute_input":"2022-04-14T14:03:06.824846Z","iopub.status.idle":"2022-04-14T14:03:06.832040Z","shell.execute_reply.started":"2022-04-14T14:03:06.824805Z","shell.execute_reply":"2022-04-14T14:03:06.831022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 由于时间是从2018.9.20-2020.9.22 任务需求预测2020.9.23~9.29的购买。\n### 考虑先利用2018.9.20-2019.8.30 作为训练集 然后2019.9.1~9.31 作为测试集 来测试模型的可行性。\n### 最终在通过训练全量的数据集最小化mse，然后生成最终的结果。","metadata":{}},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:06.833208Z","iopub.execute_input":"2022-04-14T14:03:06.833453Z","iopub.status.idle":"2022-04-14T14:03:06.848973Z","shell.execute_reply.started":"2022-04-14T14:03:06.833425Z","shell.execute_reply":"2022-04-14T14:03:06.847998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## transactions_train.csv Done!","metadata":{}},{"cell_type":"markdown","source":"## 观察数据 是否有季节影响","metadata":{}},{"cell_type":"code","source":"article_code.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:06.850512Z","iopub.execute_input":"2022-04-14T14:03:06.850764Z","iopub.status.idle":"2022-04-14T14:03:06.863741Z","shell.execute_reply.started":"2022-04-14T14:03:06.850737Z","shell.execute_reply":"2022-04-14T14:03:06.862890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_dict = {}\nfor dat in transactions.t_dat.unique():\n    time_dict[dat] = int(dat[5:7])\nt_dat_map = transactions.t_dat.copy()\nt_dat_map = t_dat_map.map(lambda x: time_dict[x])\n\nproduct_type_uni = article_code.product_type_no.unique()\nproduct_group_uni = article_code.product_group_name.unique()\nproduct_name_uni = article.prod_name.unique()\n\narticle2type = {}\nfor i, key in enumerate(article_code.article_id):\n    article2type[key] = article_code.product_type_no[i]\nproduct_type_size = len(article_code.product_type_no.unique())\n\narticle2group = {}\nfor i, key in enumerate(article_code.article_id):\n    article2group[key] = article_code.product_group_name[i]\nproduct_group_size = len(article_code.product_group_name.unique())\n\narticle2name = {}\nfor i, key in enumerate(article_code.article_id):\n    article2name[key] = article.prod_name[i]\nprod_name_size = len(article.prod_name.unique())","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:06.865172Z","iopub.execute_input":"2022-04-14T14:03:06.866010Z","iopub.status.idle":"2022-04-14T14:03:23.109026Z","shell.execute_reply.started":"2022-04-14T14:03:06.865921Z","shell.execute_reply":"2022-04-14T14:03:23.107881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_month_data(data, month, t_dat_map = t_dat_map):\n    month_data = data.article_id[t_dat_map == month]\n    return month_data\ndef give_month_get_res(month_data, product_uni = product_type_uni, dict = article2type):\n    month_data = month_data.map(lambda x: dict[x])\n    res = []\n    for i in product_uni:\n        res.append( (month_data==i).sum() )\n    return res\n\ndef new_give_month_get_res(month_data, product_uni = product_type_uni, dict = article2type):\n    month_data = month_data.map(lambda x: dict[x])\n    res = []\n    v_counts = month_data.value_counts()\n    for i in product_uni:\n        if i in v_counts.index:\n            res.append( v_counts[i] )\n        else:\n            res.append( 0 )\n    return res\n\n#month_tmp = get_month_data(transactions, 1)\n#res_tmp = give_month_get_res(month_tmp)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.110428Z","iopub.execute_input":"2022-04-14T14:03:23.110726Z","iopub.status.idle":"2022-04-14T14:03:23.118255Z","shell.execute_reply.started":"2022-04-14T14:03:23.110686Z","shell.execute_reply":"2022-04-14T14:03:23.117483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 经观察发现 prod_name 即商品名公用45875种，如果只用product_type_name(132种)来衡量商品的话就显得不太充分。\n### Notes: product_type_name = 'Dress'， 共有5165种prod_name与之相关联。","metadata":{}},{"cell_type":"code","source":"import time\nfig, axs = plt.subplots(3, 4, figsize=(36,12))\nfor i in range(12):\n    row, col = int(i/4), i%4\n    month_data = get_month_data(transactions, i+1)\n    res = new_give_month_get_res(month_data, product_uni = product_name_uni, dict = article2name)\n    axs[row, col].bar(list(range(40000)), height = res[:40000])\n    axs[row, col].set_title(f\"Month: {i+1} for prod name\")\n    print(f'fig: {i} done at {time.asctime()}')\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.694096Z","iopub.status.idle":"2022-04-14T14:03:23.694736Z","shell.execute_reply.started":"2022-04-14T14:03:23.694455Z","shell.execute_reply":"2022-04-14T14:03:23.694492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## As we can see, distribution of about prod_name in month are many different!","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, axs = plt.subplots(3, 4, figsize=(36,12))\nfor i in range(12):\n    row, col = int(i/4), i%4\n    month_data = get_month_data(transactions, i+1)\n    res = new_give_month_get_res(month_data)\n    axs[row, col].bar(product_type_uni[:80], height = res[:80])\n    axs[row, col].set_title(f\"Month: {i+1} for product type\")\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.696206Z","iopub.status.idle":"2022-04-14T14:03:23.696524Z","shell.execute_reply.started":"2022-04-14T14:03:23.696370Z","shell.execute_reply":"2022-04-14T14:03:23.696387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## As we can see, distribution of month 9, 10, 11 are more similar","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(3, 4, figsize=(36,12))\nfor i in range(12):\n    row, col = int(i/4), i%4\n    month_data = get_month_data(transactions, i+1)\n    res = give_month_get_res(month_data, product_uni = product_group_uni, dict = article2group)\n    axs[row, col].bar(product_group_uni, height = res)\n    axs[row, col].set_title(f\"Month: {i+1} for product group\")\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.697729Z","iopub.status.idle":"2022-04-14T14:03:23.698278Z","shell.execute_reply.started":"2022-04-14T14:03:23.698102Z","shell.execute_reply":"2022-04-14T14:03:23.698122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## can't find any userful information about product group name","metadata":{}},{"cell_type":"markdown","source":"## 用户会重复购买吗？先观察下用户的消费情况！","metadata":{}},{"cell_type":"code","source":"user_trans = transactions.groupby('customer_id')","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.699338Z","iopub.status.idle":"2022-04-14T14:03:23.699840Z","shell.execute_reply.started":"2022-04-14T14:03:23.699673Z","shell.execute_reply":"2022-04-14T14:03:23.699692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"repeat_buy_num = []\ntotal_buy_num = []\nfor name, g in user_trans:\n    total_buy_num.append(g.shape[0])\n    tmp = pd.value_counts(g.article_id)\n    repeat_buy_num.append(tmp.index[tmp>1].to_list())","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.700783Z","iopub.status.idle":"2022-04-14T14:03:23.701282Z","shell.execute_reply.started":"2022-04-14T14:03:23.701094Z","shell.execute_reply":"2022-04-14T14:03:23.701132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('process_0/repeat_buy_num.pkl','wb') as f:\n    pickle.dump(repeat_buy_num,f)\nwith open('process_0/total_buy_num.pkl','wb') as f:\n    pickle.dump(total_buy_num,f)\n\n'''\nwith open('process_0/repeat_buy_num.pkl','rb') as f:\n    tmp = pickle.load(f)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.702237Z","iopub.status.idle":"2022-04-14T14:03:23.702766Z","shell.execute_reply.started":"2022-04-14T14:03:23.702579Z","shell.execute_reply":"2022-04-14T14:03:23.702598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_buy_pd = pd.Series(total_buy_num)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.703671Z","iopub.status.idle":"2022-04-14T14:03:23.704171Z","shell.execute_reply.started":"2022-04-14T14:03:23.704009Z","shell.execute_reply":"2022-04-14T14:03:23.704028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## there are 131514 个user only buy once\n## 31788324 条购买记录中有 3515413条重复购买记录 （11%）","metadata":{}},{"cell_type":"code","source":"(total_buy_pd == 1).sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.705112Z","iopub.status.idle":"2022-04-14T14:03:23.705638Z","shell.execute_reply.started":"2022-04-14T14:03:23.705467Z","shell.execute_reply":"2022-04-14T14:03:23.705488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_buy_pd.sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.706578Z","iopub.status.idle":"2022-04-14T14:03:23.707077Z","shell.execute_reply.started":"2022-04-14T14:03:23.706903Z","shell.execute_reply":"2022-04-14T14:03:23.706921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_count = [0] * len(article_code.product_type_no.unique())\nlen(type_count)","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.707973Z","iopub.status.idle":"2022-04-14T14:03:23.708502Z","shell.execute_reply.started":"2022-04-14T14:03:23.708334Z","shell.execute_reply":"2022-04-14T14:03:23.708354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_count = [0] * len(article_code.product_type_no.unique())\nfor i in repeat_buy_num:\n    for article_id in i:\n        #print(f'article_id: {article_id} type_no: {article2type[article_id]}')\n        type_count[article2type[article_id]] += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.709411Z","iopub.status.idle":"2022-04-14T14:03:23.709891Z","shell.execute_reply.started":"2022-04-14T14:03:23.709709Z","shell.execute_reply":"2022-04-14T14:03:23.709727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,type in enumerate(type_count):\n    print( i, type)\n    if i == 80: break","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.710774Z","iopub.status.idle":"2022-04-14T14:03:23.711082Z","shell.execute_reply.started":"2022-04-14T14:03:23.710919Z","shell.execute_reply":"2022-04-14T14:03:23.710941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(product_type_uni[:80], height = type_count[:80])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.712202Z","iopub.status.idle":"2022-04-14T14:03:23.712502Z","shell.execute_reply.started":"2022-04-14T14:03:23.712355Z","shell.execute_reply":"2022-04-14T14:03:23.712371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.713474Z","iopub.status.idle":"2022-04-14T14:03:23.713966Z","shell.execute_reply.started":"2022-04-14T14:03:23.713800Z","shell.execute_reply":"2022-04-14T14:03:23.713821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article[article.product_type_name == 'Dress'].prod_name.unique().shape","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.714856Z","iopub.status.idle":"2022-04-14T14:03:23.715381Z","shell.execute_reply.started":"2022-04-14T14:03:23.715186Z","shell.execute_reply":"2022-04-14T14:03:23.715205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article.prod_name.unique().shape","metadata":{"execution":{"iopub.status.busy":"2022-04-14T14:03:23.716280Z","iopub.status.idle":"2022-04-14T14:03:23.716827Z","shell.execute_reply.started":"2022-04-14T14:03:23.716619Z","shell.execute_reply":"2022-04-14T14:03:23.716657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}