{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-21T03:00:12.438381Z","iopub.execute_input":"2022-06-21T03:00:12.439076Z","iopub.status.idle":"2022-06-21T03:00:12.445349Z","shell.execute_reply.started":"2022-06-21T03:00:12.439036Z","shell.execute_reply":"2022-06-21T03:00:12.444427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 首先对商品和人购买的情况进行统计，过滤出热门的商品或者是人，同时统计出商品被购买的次数（或者是人购买的次数）\nfrom tqdm import tqdm\nimport csv\n\ndef statisitc_counter(path):\n    articles_number_dict = {}\n    customer_number_dict = {}\n    \n    csv_reader = csv.reader(open(path))\n    for item in tqdm(csv_reader):\n        if item[1] not in customer_number_dict.keys():\n            customer_number_dict[item[1]]  = 1\n        else:\n            customer_number_dict[item[1]] += 1\n        \n        if item[2] not in articles_number_dict.keys():\n            articles_number_dict[item[2]] = 1\n        else:\n            articles_number_dict[item[2]] += 1\n    \n    return articles_number_dict,customer_number_dict","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:00:12.447706Z","iopub.execute_input":"2022-06-21T03:00:12.448424Z","iopub.status.idle":"2022-06-21T03:00:12.456672Z","shell.execute_reply.started":"2022-06-21T03:00:12.448380Z","shell.execute_reply":"2022-06-21T03:00:12.455819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 整个过程大概需要三分多钟\n\narticles_number_dict,customer_number_dict = statisitc_counter(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:00:12.458126Z","iopub.execute_input":"2022-06-21T03:00:12.458751Z","iopub.status.idle":"2022-06-21T03:03:03.153684Z","shell.execute_reply.started":"2022-06-21T03:00:12.458718Z","shell.execute_reply":"2022-06-21T03:03:03.152598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_articles_number_dict = sorted(articles_number_dict.items(), key=lambda x: x[1], reverse=True)\nsorted_customer_number_dict = sorted(customer_number_dict.items(), key=lambda x: x[1], reverse=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:03.155494Z","iopub.execute_input":"2022-06-21T03:03:03.156387Z","iopub.status.idle":"2022-06-21T03:03:04.030063Z","shell.execute_reply.started":"2022-06-21T03:03:03.156335Z","shell.execute_reply":"2022-06-21T03:03:04.029071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 有被购买过的商品的总数是104548\n# 购买过商品的人的总数是1362282\n\nprint(len(sorted_articles_number_dict))\nprint(len(sorted_customer_number_dict))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:04.034173Z","iopub.execute_input":"2022-06-21T03:03:04.034525Z","iopub.status.idle":"2022-06-21T03:03:04.040497Z","shell.execute_reply.started":"2022-06-21T03:03:04.034493Z","shell.execute_reply":"2022-06-21T03:03:04.039225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 绘制一下两者的折线图\n\narticles_list,articles_number_list = zip(*sorted_articles_number_dict)\ncustomer_list,customer_number_list = zip(*sorted_customer_number_dict)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:04.042227Z","iopub.execute_input":"2022-06-21T03:03:04.042699Z","iopub.status.idle":"2022-06-21T03:03:12.386001Z","shell.execute_reply.started":"2022-06-21T03:03:04.042638Z","shell.execute_reply":"2022-06-21T03:03:12.384881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 建一个文件夹保存图片，以方便撰写报告\nimport os\ntry:\n    os.mkdir(\"./figures\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:12.387285Z","iopub.execute_input":"2022-06-21T03:03:12.387622Z","iopub.status.idle":"2022-06-21T03:03:12.393483Z","shell.execute_reply.started":"2022-06-21T03:03:12.387590Z","shell.execute_reply":"2022-06-21T03:03:12.392285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 先来绘制一下针对商品的被购买次数-排名位次折线图\n# 这张图可以在Output的figures文件夹下找到\nimport matplotlib.pyplot as plt\nfrom pylab import *   \n\nx = range(len(articles_number_list))\ny = articles_number_list\nplt.title(\"Distribution of purchased articles\") #标题\nplt.xlabel(u\"articles\") #X轴标签\nplt.ylabel(\"purchased times\") #Y轴标签\nplt.plot(x, y,label=\"articles\")\nplt.savefig(\"./figures/Distribution of purchased articles.jpg\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:31:36.533465Z","iopub.execute_input":"2022-06-21T03:31:36.534007Z","iopub.status.idle":"2022-06-21T03:31:36.868741Z","shell.execute_reply.started":"2022-06-21T03:31:36.533972Z","shell.execute_reply":"2022-06-21T03:31:36.867509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 然后是消费者消费的次数分布\nimport matplotlib.pyplot as plt\nfrom pylab import *   \n\nx = range(len(customer_number_list))\ny = customer_number_list\nplt.title(\"Distribution of consume times\") #标题\nplt.xlabel(u\"customer\") #X轴标签\nplt.ylabel(\"consume times\") #Y轴标签\nplt.plot(x, y,label=\"customer\")\nplt.savefig(\"./figures/Distribution of consume times.jpg\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:31:38.158889Z","iopub.execute_input":"2022-06-21T03:31:38.159478Z","iopub.status.idle":"2022-06-21T03:31:39.245305Z","shell.execute_reply.started":"2022-06-21T03:31:38.159444Z","shell.execute_reply":"2022-06-21T03:31:39.244481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 可以看到，其基本是满足长尾分布的","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.551965Z","iopub.execute_input":"2022-06-21T03:03:13.552517Z","iopub.status.idle":"2022-06-21T03:03:13.556478Z","shell.execute_reply.started":"2022-06-21T03:03:13.552483Z","shell.execute_reply":"2022-06-21T03:03:13.555760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 然后，我们来统计一下前%k的商品或者消费者的相关次数\n\ndef show_top_proportion_k(target_list,total_proportion = 10,step = 10):\n    # 本函数默认显示前5%的相关次数，精度默认按照0.1%的来\n    length = len(target_list)\n    \n    for i in range(0,total_proportion * step):\n        print(\"number of top {}% is:{}\".format(i/step,target_list[int(length * i /(step * 100))]))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.557829Z","iopub.execute_input":"2022-06-21T03:03:13.558230Z","iopub.status.idle":"2022-06-21T03:03:13.568916Z","shell.execute_reply.started":"2022-06-21T03:03:13.558199Z","shell.execute_reply":"2022-06-21T03:03:13.568092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 看看商品被购买数量的分布情况\nshow_top_proportion_k(articles_number_list)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.570012Z","iopub.execute_input":"2022-06-21T03:03:13.570590Z","iopub.status.idle":"2022-06-21T03:03:13.580092Z","shell.execute_reply.started":"2022-06-21T03:03:13.570528Z","shell.execute_reply":"2022-06-21T03:03:13.579098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 看看用户购买数量的分布情况\nshow_top_proportion_k(customer_number_list)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.581804Z","iopub.execute_input":"2022-06-21T03:03:13.582595Z","iopub.status.idle":"2022-06-21T03:03:13.590564Z","shell.execute_reply.started":"2022-06-21T03:03:13.582521Z","shell.execute_reply":"2022-06-21T03:03:13.589369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 之后，我们分别计算占所有被购买商品50%的商品数量以及购买了50%商品的人的数量，并且将它们作为数据的高频部分\n\ndef count_proportion(target_list,proportion = 0.5):\n    sum = 0 \n    length = len(target_list)\n    for i in range(length):\n        sum += target_list[i]\n    target_sum = sum * proportion\n    sum = 0\n    for i in range(length):\n        sum += target_list[i]\n        if sum > target_sum:\n            return i","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.592057Z","iopub.execute_input":"2022-06-21T03:03:13.592970Z","iopub.status.idle":"2022-06-21T03:03:13.599596Z","shell.execute_reply.started":"2022-06-21T03:03:13.592932Z","shell.execute_reply":"2022-06-21T03:03:13.598836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_count = count_proportion(articles_number_list)\ncustomer_count = count_proportion(customer_number_list,proportion = 0.1)\n\n\n# 这里因为用户的数量比较多，为了将筛选出来的商品和用户的数量都控制在一万及一下\nprint(\"占商品购买次数50%的商品index是{}，占比{}\".format(articles_count,articles_count/len(articles_number_list)))\nprint(\"占用户购买次数10%的用户index是{}，占比{}\".format(customer_count,customer_count/len(customer_number_list)))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.600590Z","iopub.execute_input":"2022-06-21T03:03:13.601634Z","iopub.status.idle":"2022-06-21T03:03:13.741147Z","shell.execute_reply.started":"2022-06-21T03:03:13.601585Z","shell.execute_reply":"2022-06-21T03:03:13.740115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_articles_set = set(articles_list[0:6800])\nfiltered_customer_set = set(customer_list[0:10946])","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.742694Z","iopub.execute_input":"2022-06-21T03:03:13.743253Z","iopub.status.idle":"2022-06-21T03:03:13.779072Z","shell.execute_reply.started":"2022-06-21T03:03:13.743203Z","shell.execute_reply":"2022-06-21T03:03:13.778276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./filtered_files\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.780125Z","iopub.execute_input":"2022-06-21T03:03:13.781018Z","iopub.status.idle":"2022-06-21T03:03:13.790167Z","shell.execute_reply.started":"2022-06-21T03:03:13.780962Z","shell.execute_reply":"2022-06-21T03:03:13.789100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_csv_reader = csv.reader(open(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"))\ncustomer_csv_reader = csv.reader(open(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\"))\n\nfiltered_articles_csv_writer = csv.writer(open(\"./filtered_files/filtered_articles.csv\",\"w\"))\nfiltered_customer_csv_writer = csv.writer(open(\"./filtered_files/filtered_customer.csv\",\"w\"))\n\nfor item in tqdm(articles_csv_reader):\n    if item[0] in filtered_articles_set:\n        filtered_articles_csv_writer.writerow(item)\nfor item in tqdm(customer_csv_reader):\n    if item[0] in filtered_customer_set:\n        filtered_customer_csv_writer.writerow(item)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:13.791540Z","iopub.execute_input":"2022-06-21T03:03:13.792068Z","iopub.status.idle":"2022-06-21T03:03:22.200653Z","shell.execute_reply.started":"2022-06-21T03:03:13.792036Z","shell.execute_reply":"2022-06-21T03:03:22.199618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 在使用关联规则挖掘之前，我们先来进行一些统计性的分析\n\n# 例如说，统计一下每一种特征出现的频率和它们在原有数据当中出现的频率，并且对比之","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:22.201941Z","iopub.execute_input":"2022-06-21T03:03:22.202342Z","iopub.status.idle":"2022-06-21T03:03:22.206716Z","shell.execute_reply.started":"2022-06-21T03:03:22.202308Z","shell.execute_reply":"2022-06-21T03:03:22.205937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 首先来构造一个统计属性-index的函数\n\ndef create_attri_index_dict(path):\n    csv_reader = csv.reader(open(path))\n    for head in csv_reader:\n        break\n    attri_index_dict = {}\n    index_attri_dict = {}\n    for i,attri in enumerate(head):\n        attri_index_dict[attri] = i\n        index_attri_dict[i] = attri\n        \n    return attri_index_dict,index_attri_dict\n\narticle_attri_index_dict,article_index_attri_dict = create_attri_index_dict(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomer_attri_index_dict,customer_index_attri_dict = create_attri_index_dict(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\n\nprint(article_attri_index_dict)\nprint(customer_attri_index_dict)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:22.207656Z","iopub.execute_input":"2022-06-21T03:03:22.208007Z","iopub.status.idle":"2022-06-21T03:03:22.222762Z","shell.execute_reply.started":"2022-06-21T03:03:22.207977Z","shell.execute_reply":"2022-06-21T03:03:22.221924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# 这个函数找出该item集合当中频数top k的项目，并且统计其数量\ndef find_high_attri(items,k= 5):\n    statistic_dict = {}\n    for item in tqdm(items):\n        if item not in statistic_dict.keys():\n            statistic_dict[item] = 1\n        else:\n            statistic_dict[item] += 1\n    sorted_list = sorted(statistic_dict.items(), key=lambda x: x[1], reverse=True)\n    \n    new_statistic_dict = {\"others\":0}\n    for i in range(0,k):\n        new_statistic_dict[sorted_list[i][0]] = sorted_list[i][1]\n    for i in range(k,len(sorted_list)):\n        new_statistic_dict[\"others\"] += sorted_list[i][1]\n    \n    return new_statistic_dict\n\ndef get_attri_items(csv_path,attri_index_dict,attri_name):\n    csv_reader = csv.reader(open(csv_path))\n    items = []\n    index = attri_index_dict[attri_name]\n    for line in tqdm(csv_reader):\n        items.append(line[index])\n        \n    return items\n\n# 生成柱状图\ndef create_bar(csv_path,attri_dict,attri_name,save_path,name,k=5):\n    items = get_attri_items(csv_path,attri_dict,attri_name)\n    high_attri_dict = find_high_attri(items,k)\n    \n    others_number = high_attri_dict.pop(\"others\")\n    \n    sorted_items =  sorted(high_attri_dict.items(), key=lambda x: x[1], reverse=True)\n    \n    if others_number != 0:\n        sorted_items.append((\"others\",others_number))\n    \n    cell,pvalue = zip(* sorted_items)\n    \n    x = cell\n    y = pvalue\n    \n    fig = plt.figure()\n    plt.bar(x,y,0.4,color=\"steelblue\")\n    \n    for a,b in zip(x,y):  \n        plt.text(a,b,'%.2f'%b,ha='center',va='bottom',fontsize=7);\n    plt.ylabel('p value')\n    \"\"\"\n    index = np.arange(len(cell))\n    \n    width = 0.30\n\n    figsize = (50,40)#调整绘制图片的比例\n    #若是不想显示直线，可以直接将上面两行注释掉\n    plt.bar(index, pvalue, width,color=\"#87CEFA\") #绘制柱状图\n    #plt.xlabel('cell type') #x轴\n    plt.ylabel('p value') #y轴\n    plt.title(name) #图像的名称\n    plt.xticks(index, cell,fontsize=5) #将横坐标用cell替换,fontsize用来调整字体的大小\n    plt.legend() #显示label\n    \n    x,y = cell,pvalue\n    for a,b in zip(x,y):   #柱子上的数字显示\n        plt.text(a,b,'%.2f'%b,ha='center',va='bottom',fontsize=7)\"\"\"\n        \n    plt.savefig(save_path,dpi = 2000) #保存图像，dpi可以调整图像的像素大小\n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:22.224646Z","iopub.execute_input":"2022-06-21T03:03:22.225017Z","iopub.status.idle":"2022-06-21T03:03:22.241761Z","shell.execute_reply.started":"2022-06-21T03:03:22.224986Z","shell.execute_reply":"2022-06-21T03:03:22.240719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(\"./filtered_files\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:22.243067Z","iopub.execute_input":"2022-06-21T03:03:22.243412Z","iopub.status.idle":"2022-06-21T03:03:22.257307Z","shell.execute_reply.started":"2022-06-21T03:03:22.243362Z","shell.execute_reply":"2022-06-21T03:03:22.256360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 这里主要对商品类型、商品的product group、商品颜色、index name进行分析展示\n\ntry:\n    os.mkdir(\"./figures/comparation\")\n    os.mkdir(\"./figures/comparation/articles\")\nexcept:\n    print(\"exist\")\n    \n    \n{'article_id': 0, 'product_code': 1, 'prod_name': 2, 'product_type_no': 3, 'product_type_name': 4, 'product_group_name': 5\n , 'graphical_appearance_no': 6, 'graphical_appearance_name': 7, 'colour_group_code': 8, 'colour_group_name': 9, \n 'perceived_colour_value_id': 10, 'perceived_colour_value_name': 11, 'perceived_colour_master_id': 12, \n 'perceived_colour_master_name': 13, 'department_no': 14, 'department_name': 15, 'index_code': 16, 'index_name': 17,\n 'index_group_no': 18, 'index_group_name': 19, 'section_no': 20, 'section_name': 21, 'garment_group_no': 22, \n 'garment_group_name': 23, 'detail_desc': 24}\n{'customer_id': 0, 'FN': 1, 'Active': 2, 'club_member_status': 3, 'fashion_news_frequency': 4, 'age': 5, 'postal_code': 6}\n\n# 展示的键包括：product_type_name \\colour_group_name \\ index_group_name\\ product_group_name\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\",\n           article_attri_index_dict,\"product_type_name\",\n           \"./figures/comparation/articles/total_article_product_type.jpg\",\n           \"distribution of product type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:22.259199Z","iopub.execute_input":"2022-06-21T03:03:22.259713Z","iopub.status.idle":"2022-06-21T03:03:28.439113Z","shell.execute_reply.started":"2022-06-21T03:03:22.259665Z","shell.execute_reply":"2022-06-21T03:03:28.438214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_bar(\"./filtered_files/filtered_articles.csv\",\n           article_attri_index_dict,\"product_type_name\",\n           \"./figures/comparation/articles/filtered_article_product_type.jpg\",\n           \"distribution of product type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:28.443450Z","iopub.execute_input":"2022-06-21T03:03:28.444883Z","iopub.status.idle":"2022-06-21T03:03:33.286062Z","shell.execute_reply.started":"2022-06-21T03:03:28.444832Z","shell.execute_reply":"2022-06-21T03:03:33.284971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 从类型的分布上来讲，感觉它们没有明显的区别","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:33.287916Z","iopub.execute_input":"2022-06-21T03:03:33.288971Z","iopub.status.idle":"2022-06-21T03:03:33.293883Z","shell.execute_reply.started":"2022-06-21T03:03:33.288920Z","shell.execute_reply":"2022-06-21T03:03:33.292601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 展示的键包括：product_type_name \\colour_group_name \\ index_group_name\\ product_group_name\n# 接下来看color的分布\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\",\n           article_attri_index_dict,\"colour_group_name\",\n           \"./figures/comparation/articles/total_article_colour_group_name.jpg\",\n           \"distribution of color type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:33.295636Z","iopub.execute_input":"2022-06-21T03:03:33.296314Z","iopub.status.idle":"2022-06-21T03:03:39.349005Z","shell.execute_reply.started":"2022-06-21T03:03:33.296265Z","shell.execute_reply":"2022-06-21T03:03:39.347619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_bar(\"./filtered_files/filtered_articles.csv\",\n           article_attri_index_dict,\"colour_group_name\",\n           \"./figures/comparation/articles/filtered_article_colour_group_name.jpg\",\n           \"distribution of color type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:39.350707Z","iopub.execute_input":"2022-06-21T03:03:39.351121Z","iopub.status.idle":"2022-06-21T03:03:44.110874Z","shell.execute_reply.started":"2022-06-21T03:03:39.351087Z","shell.execute_reply":"2022-06-21T03:03:44.109632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"从上述的颜色对比来看，我们可以发现其实黑白更加受到欢迎。例如，黑色在热销的商品当中的占比是高于它在所有商品当中占比的。","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 展示的键包括：product_type_name \\colour_group_name \\ index_group_name\\ product_group_name\n# 接下来看index的分布\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\",\n           article_attri_index_dict,\"index_group_name\",\n           \"./figures/comparation/articles/total_article_index_group_name.jpg\",\n           \"distribution of index type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:44.112472Z","iopub.execute_input":"2022-06-21T03:03:44.112855Z","iopub.status.idle":"2022-06-21T03:03:50.142194Z","shell.execute_reply.started":"2022-06-21T03:03:44.112823Z","shell.execute_reply":"2022-06-21T03:03:50.141238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_bar(\"./filtered_files/filtered_articles.csv\",\n           article_attri_index_dict,\"index_group_name\",\n           \"./figures/comparation/articles/filtered_article_index_group_name.jpg\",\n           \"distribution of index type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:50.143513Z","iopub.execute_input":"2022-06-21T03:03:50.143882Z","iopub.status.idle":"2022-06-21T03:03:54.856706Z","shell.execute_reply.started":"2022-06-21T03:03:50.143850Z","shell.execute_reply":"2022-06-21T03:03:54.855768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"可以看到，index的分布基本也有比较大的变化","metadata":{}},{"cell_type":"code","source":"# 展示的键包括：product_type_name \\colour_group_name \\ index_group_name\\ product_group_name\n# 接下来看product_group的分布\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\",\n           article_attri_index_dict,\"product_group_name\",\n           \"./figures/comparation/articles/total_article_product_group_name.jpg\",\n           \"distribution of product type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:03:54.859697Z","iopub.execute_input":"2022-06-21T03:03:54.860030Z","iopub.status.idle":"2022-06-21T03:04:00.939032Z","shell.execute_reply.started":"2022-06-21T03:03:54.859999Z","shell.execute_reply":"2022-06-21T03:04:00.936524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_bar(\"./filtered_files/filtered_articles.csv\",\n           article_attri_index_dict,\"product_group_name\",\n           \"./figures/comparation/articles/filtered_article_product_group_name.jpg\",\n           \"distribution of product type\",\n           k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:00.940733Z","iopub.execute_input":"2022-06-21T03:04:00.941400Z","iopub.status.idle":"2022-06-21T03:04:05.566395Z","shell.execute_reply.started":"2022-06-21T03:04:00.941349Z","shell.execute_reply":"2022-06-21T03:04:05.565373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"product group也有较大的差别","metadata":{}},{"cell_type":"code","source":"# 接下来对customer的情况进行统计和分析","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:05.568118Z","iopub.execute_input":"2022-06-21T03:04:05.568464Z","iopub.status.idle":"2022-06-21T03:04:05.572030Z","shell.execute_reply.started":"2022-06-21T03:04:05.568432Z","shell.execute_reply":"2022-06-21T03:04:05.571169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_attri_index_dict","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:05.573424Z","iopub.execute_input":"2022-06-21T03:04:05.573999Z","iopub.status.idle":"2022-06-21T03:04:05.589707Z","shell.execute_reply.started":"2022-06-21T03:04:05.573966Z","shell.execute_reply":"2022-06-21T03:04:05.588771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./figures/comparation/customers\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:05.591316Z","iopub.execute_input":"2022-06-21T03:04:05.591728Z","iopub.status.idle":"2022-06-21T03:04:05.599003Z","shell.execute_reply.started":"2022-06-21T03:04:05.591696Z","shell.execute_reply":"2022-06-21T03:04:05.598157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 展示的键包括：club_member_status \\ fashion_news_frequency\n# 接下来看club_member_status的分布\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\",\n           customer_attri_index_dict,\"club_member_status\",\n           \"./figures/comparation/customers/total_customers_club_member_status.jpg\",\n           \"distribution of club status\",\n           k=3)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:05.600500Z","iopub.execute_input":"2022-06-21T03:04:05.601525Z","iopub.status.idle":"2022-06-21T03:04:17.826069Z","shell.execute_reply.started":"2022-06-21T03:04:05.601462Z","shell.execute_reply":"2022-06-21T03:04:17.824900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncreate_bar(\"./filtered_files/filtered_customer.csv\",\n           customer_attri_index_dict,\"club_member_status\",\n           \"./figures/comparation/customers/filtered_customers_club_member_status.jpg\",\n           \"distribution of club status\",\n           k=3)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:17.827777Z","iopub.execute_input":"2022-06-21T03:04:17.828280Z","iopub.status.idle":"2022-06-21T03:04:22.562686Z","shell.execute_reply.started":"2022-06-21T03:04:17.828231Z","shell.execute_reply":"2022-06-21T03:04:22.561716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 可以看到，购买行为较为活跃的人当中，还是active的人最多","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:22.563976Z","iopub.execute_input":"2022-06-21T03:04:22.564423Z","iopub.status.idle":"2022-06-21T03:04:22.569134Z","shell.execute_reply.started":"2022-06-21T03:04:22.564328Z","shell.execute_reply":"2022-06-21T03:04:22.567959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 展示的键包括：club_member_status \\ fashion_news_frequency\n# 接下来看club_member_status的分布\ncreate_bar(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\",\n           customer_attri_index_dict,\"fashion_news_frequency\",\n           \"./figures/comparation/customers/total_customers_fashion_news_frequency.jpg\",\n           \"distribution of fashio news frequence\",\n           k=4)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:22.570405Z","iopub.execute_input":"2022-06-21T03:04:22.570905Z","iopub.status.idle":"2022-06-21T03:04:34.852955Z","shell.execute_reply.started":"2022-06-21T03:04:22.570846Z","shell.execute_reply":"2022-06-21T03:04:34.851870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncreate_bar(\"./filtered_files/filtered_customer.csv\",\n           customer_attri_index_dict,\"fashion_news_frequency\",\n           \"./figures/comparation/customers/filtered_customers_fashion_news_frequency.jpg\",\n           \"distribution of fashion_news_frequency\",\n           k=2)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:34.854238Z","iopub.execute_input":"2022-06-21T03:04:34.854620Z","iopub.status.idle":"2022-06-21T03:04:39.449927Z","shell.execute_reply.started":"2022-06-21T03:04:34.854577Z","shell.execute_reply":"2022-06-21T03:04:39.448876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 以上是商品和用户的特征行为，下面来进行频繁项集挖掘","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:39.451046Z","iopub.execute_input":"2022-06-21T03:04:39.451360Z","iopub.status.idle":"2022-06-21T03:04:39.456296Z","shell.execute_reply.started":"2022-06-21T03:04:39.451330Z","shell.execute_reply":"2022-06-21T03:04:39.455321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficient-apriori","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:39.457457Z","iopub.execute_input":"2022-06-21T03:04:39.458437Z","iopub.status.idle":"2022-06-21T03:04:52.832006Z","shell.execute_reply.started":"2022-06-21T03:04:39.458402Z","shell.execute_reply":"2022-06-21T03:04:52.830946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficient_apriori import apriori \n\n\n\n# 我感觉这里的置信度不太重要，仅需要通过支持度来筛选一下即可\ndef mining(path,min_support = 0.2,min_confidence = 1):\n    csv_reader = csv.reader(open(path))\n    data = []\n    for item in csv_reader:\n        data.append(tuple(item))\n    itemsets, rules = apriori(data,min_support=min_support,min_confidence=min_confidence)\n    \n    return itemsets,rules","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:52.834276Z","iopub.execute_input":"2022-06-21T03:04:52.834852Z","iopub.status.idle":"2022-06-21T03:04:52.843393Z","shell.execute_reply.started":"2022-06-21T03:04:52.834799Z","shell.execute_reply":"2022-06-21T03:04:52.842070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 首先来对article的频繁项集进行挖掘\n\nitemsets,rules = mining(\"./filtered_files/filtered_articles.csv\")\nprint(rules)\nprint(itemsets)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:52.845245Z","iopub.execute_input":"2022-06-21T03:04:52.845643Z","iopub.status.idle":"2022-06-21T03:04:53.313358Z","shell.execute_reply.started":"2022-06-21T03:04:52.845595Z","shell.execute_reply":"2022-06-21T03:04:53.312272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 以上的项集非常不好解读，我们采取一些过滤规则来筛选一下，主要还是筛除不好解读的数字","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:53.314722Z","iopub.execute_input":"2022-06-21T03:04:53.315164Z","iopub.status.idle":"2022-06-21T03:04:53.320248Z","shell.execute_reply.started":"2022-06-21T03:04:53.315131Z","shell.execute_reply":"2022-06-21T03:04:53.319044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def item_filter(items_dict):\n    filtered_items = {}\n    items = items_dict.keys()\n    \n    for item in tqdm(items):\n        if len(items) < 3:\n            continue\n        temp_item = []\n        for value in item:\n            try:\n                float(value)\n            except:\n                if len(value) > 2:\n                    temp_item.append(value)\n        if len(temp_item) < 2:\n            continue\n        filtered_items[tuple(temp_item)] = items_dict[item]\n    return filtered_items","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:53.321933Z","iopub.execute_input":"2022-06-21T03:04:53.322378Z","iopub.status.idle":"2022-06-21T03:04:53.331632Z","shell.execute_reply.started":"2022-06-21T03:04:53.322344Z","shell.execute_reply":"2022-06-21T03:04:53.330436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(itemsets)\nprint(itemsets.keys())\nitemsets[2]\nitem_dict = {}\nfor key in itemsets.keys():\n    item_dict = {**item_dict,**itemsets[key]}","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:53.333688Z","iopub.execute_input":"2022-06-21T03:04:53.334303Z","iopub.status.idle":"2022-06-21T03:04:53.343128Z","shell.execute_reply.started":"2022-06-21T03:04:53.334270Z","shell.execute_reply":"2022-06-21T03:04:53.341850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_items = item_filter(item_dict)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:53.344817Z","iopub.execute_input":"2022-06-21T03:04:53.346109Z","iopub.status.idle":"2022-06-21T03:04:55.594528Z","shell.execute_reply.started":"2022-06-21T03:04:53.346054Z","shell.execute_reply":"2022-06-21T03:04:55.593463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_items","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:55.597610Z","iopub.execute_input":"2022-06-21T03:04:55.597918Z","iopub.status.idle":"2022-06-21T03:04:55.605859Z","shell.execute_reply.started":"2022-06-21T03:04:55.597887Z","shell.execute_reply":"2022-06-21T03:04:55.604768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 接下来开始探究什么样的商品容易被一起购买\n\ndef create_user_items(path = \"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\"):\n    user_items_dict = {}\n    csv_reader = csv.reader(open(path))\n    for item in tqdm(csv_reader):\n        if item[1] not in user_items_dict.keys():\n\n            user_items_dict[item[1]] = [item[2]]\n        else:\n            user_items_dict[item[1]].append(item[2])\n    \n    return user_items_dict","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:55.607519Z","iopub.execute_input":"2022-06-21T03:04:55.608365Z","iopub.status.idle":"2022-06-21T03:04:55.617116Z","shell.execute_reply.started":"2022-06-21T03:04:55.608318Z","shell.execute_reply":"2022-06-21T03:04:55.616294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_items_dict = create_user_items()","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:04:55.618336Z","iopub.execute_input":"2022-06-21T03:04:55.618874Z","iopub.status.idle":"2022-06-21T03:07:31.011852Z","shell.execute_reply.started":"2022-06-21T03:04:55.618842Z","shell.execute_reply":"2022-06-21T03:07:31.010731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(user_items_dict.keys()))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:07:31.013787Z","iopub.execute_input":"2022-06-21T03:07:31.014239Z","iopub.status.idle":"2022-06-21T03:07:31.020902Z","shell.execute_reply.started":"2022-06-21T03:07:31.014205Z","shell.execute_reply":"2022-06-21T03:07:31.019709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 统计一下长度分布\n\nlength_dict = {}\n\nfor key in user_items_dict.keys():\n    items = user_items_dict[key]\n    length = len(items)\n    if length not in length_dict.keys():\n        length_dict[length] = 1\n    else:\n        length_dict[length] += 1\nlength_distribution = sorted(length_dict.items(), key=lambda x: x[1], reverse=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:07:31.022531Z","iopub.execute_input":"2022-06-21T03:07:31.022956Z","iopub.status.idle":"2022-06-21T03:07:32.261016Z","shell.execute_reply.started":"2022-06-21T03:07:31.022913Z","shell.execute_reply":"2022-06-21T03:07:32.259837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(length_distribution)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:07:32.262437Z","iopub.execute_input":"2022-06-21T03:07:32.262874Z","iopub.status.idle":"2022-06-21T03:07:32.268776Z","shell.execute_reply.started":"2022-06-21T03:07:32.262840Z","shell.execute_reply":"2022-06-21T03:07:32.267704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 过滤一下长度较小的items。如果一个user购买的次数小于200，则不进行统计\nfiltered_items = []\n\nfor key in user_items_dict.keys():\n    if len(user_items_dict[key]) < 200:\n        continue\n    else:\n        filtered_items.append(user_items_dict[key])\n\nprint(len(filtered_items))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:07:32.270253Z","iopub.execute_input":"2022-06-21T03:07:32.270668Z","iopub.status.idle":"2022-06-21T03:07:32.943596Z","shell.execute_reply.started":"2022-06-21T03:07:32.270636Z","shell.execute_reply":"2022-06-21T03:07:32.942563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemsets, rules = apriori(filtered_items,min_support=0.01,min_confidence=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:07:32.945368Z","iopub.execute_input":"2022-06-21T03:07:32.945788Z","iopub.status.idle":"2022-06-21T03:09:21.615623Z","shell.execute_reply.started":"2022-06-21T03:07:32.945756Z","shell.execute_reply":"2022-06-21T03:09:21.614577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(itemsets.keys())\nprint(len(itemsets[3]))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:21.617155Z","iopub.execute_input":"2022-06-21T03:09:21.617667Z","iopub.status.idle":"2022-06-21T03:09:21.624304Z","shell.execute_reply.started":"2022-06-21T03:09:21.617618Z","shell.execute_reply":"2022-06-21T03:09:21.623257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemsets[3]","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:21.625779Z","iopub.execute_input":"2022-06-21T03:09:21.626105Z","iopub.status.idle":"2022-06-21T03:09:21.637326Z","shell.execute_reply.started":"2022-06-21T03:09:21.626075Z","shell.execute_reply":"2022-06-21T03:09:21.636603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(itemsets[2])","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:21.638501Z","iopub.execute_input":"2022-06-21T03:09:21.639145Z","iopub.status.idle":"2022-06-21T03:09:21.646692Z","shell.execute_reply.started":"2022-06-21T03:09:21.639109Z","shell.execute_reply":"2022-06-21T03:09:21.645790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_infor_dict = {}\n\ncsv_reader = csv.reader(open(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"))\nheader_index = []\nheader = True\n\nfor item in tqdm(csv_reader):\n    if header:\n        header_index = item[1:]\n        header = False\n    article_infor_dict[item[0]] = item[1:]","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:21.647915Z","iopub.execute_input":"2022-06-21T03:09:21.648663Z","iopub.status.idle":"2022-06-21T03:09:23.256693Z","shell.execute_reply.started":"2022-06-21T03:09:21.648629Z","shell.execute_reply":"2022-06-21T03:09:23.255896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze_one_item(item):\n    item_infor_list = []\n    if 1:\n        infor = article_infor_dict[item]\n        for i,attri in enumerate(header_index):\n            item_infor_list.append(attri+\":\"+infor[i])\n    return item_infor_list","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.257674Z","iopub.execute_input":"2022-06-21T03:09:23.257987Z","iopub.status.idle":"2022-06-21T03:09:23.263424Z","shell.execute_reply.started":"2022-06-21T03:09:23.257956Z","shell.execute_reply":"2022-06-21T03:09:23.262601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(analyze_one_item('0706016002'))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.264292Z","iopub.execute_input":"2022-06-21T03:09:23.264669Z","iopub.status.idle":"2022-06-21T03:09:23.275414Z","shell.execute_reply.started":"2022-06-21T03:09:23.264633Z","shell.execute_reply":"2022-06-21T03:09:23.274415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 之后来分析一下所指内容不同的item的情况\n\ndef judge(infor1,infor2):\n    if infor1[0] == infor2[0]:\n        return True\n    else:\n        return False\n            \ndef analyze_items(items):\n    items_infor = []\n    first_item_infor = []\n    first = True\n    all_same = True\n    \n    for item in items:\n        if first:\n            first_item_infor = analyze_one_item(item)\n            first = False\n        items_infor.append(analyze_one_item(item))\n        if not judge(analyze_one_item(item), first_item_infor):\n            all_same = False\n            \n    return items_infor,all_same    ","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.276481Z","iopub.execute_input":"2022-06-21T03:09:23.277106Z","iopub.status.idle":"2022-06-21T03:09:23.284398Z","shell.execute_reply.started":"2022-06-21T03:09:23.277069Z","shell.execute_reply":"2022-06-21T03:09:23.283609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items_infor,all_same = analyze_items(('0806388001', '0806388002', '0806388003'))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.285199Z","iopub.execute_input":"2022-06-21T03:09:23.285502Z","iopub.status.idle":"2022-06-21T03:09:23.298395Z","shell.execute_reply.started":"2022-06-21T03:09:23.285469Z","shell.execute_reply":"2022-06-21T03:09:23.297602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_same","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.299282Z","iopub.execute_input":"2022-06-21T03:09:23.299700Z","iopub.status.idle":"2022-06-21T03:09:23.310718Z","shell.execute_reply.started":"2022-06-21T03:09:23.299670Z","shell.execute_reply":"2022-06-21T03:09:23.309626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(itemsets[2]))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.312227Z","iopub.execute_input":"2022-06-21T03:09:23.312579Z","iopub.status.idle":"2022-06-21T03:09:23.318989Z","shell.execute_reply.started":"2022-06-21T03:09:23.312525Z","shell.execute_reply":"2022-06-21T03:09:23.317896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_itemsets_infor = []\nfor key in itemsets[2].keys():\n    itemsets_infor,same = analyze_items(key)\n    if not same:\n        filtered_itemsets_infor.append(itemsets_infor)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.320247Z","iopub.execute_input":"2022-06-21T03:09:23.320661Z","iopub.status.idle":"2022-06-21T03:09:23.343948Z","shell.execute_reply.started":"2022-06-21T03:09:23.320629Z","shell.execute_reply":"2022-06-21T03:09:23.342897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(filtered_itemsets_infor)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.345631Z","iopub.execute_input":"2022-06-21T03:09:23.346640Z","iopub.status.idle":"2022-06-21T03:09:23.353271Z","shell.execute_reply.started":"2022-06-21T03:09:23.346588Z","shell.execute_reply":"2022-06-21T03:09:23.352503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_3itemsets_infor = []\nfor key in itemsets[3].keys():\n    itemsets_infor,same = analyze_items(key)\n    if not same:\n        filtered_3itemsets_infor.append(itemsets_infor)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.354385Z","iopub.execute_input":"2022-06-21T03:09:23.355213Z","iopub.status.idle":"2022-06-21T03:09:23.362618Z","shell.execute_reply.started":"2022-06-21T03:09:23.355179Z","shell.execute_reply":"2022-06-21T03:09:23.361874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(filtered_3itemsets_infor)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.363490Z","iopub.execute_input":"2022-06-21T03:09:23.363985Z","iopub.status.idle":"2022-06-21T03:09:23.373479Z","shell.execute_reply.started":"2022-06-21T03:09:23.363954Z","shell.execute_reply":"2022-06-21T03:09:23.372720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for item in filtered_3itemsets_infor:\n    for line in zip(*item):\n        print(line)\n    print(\"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.374654Z","iopub.execute_input":"2022-06-21T03:09:23.375290Z","iopub.status.idle":"2022-06-21T03:09:23.384201Z","shell.execute_reply.started":"2022-06-21T03:09:23.375243Z","shell.execute_reply":"2022-06-21T03:09:23.383267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 下面来分析不同种类客户的购买习惯，这里主要还是进行统计性的分析\n# 我们首先根据高频用户的特征，将高频的用户找出来，按照三个因素进行划分：\n# 展示的键包括：club_member_status \\ fashion_news_frequency \\ 年龄\n# 年龄，这个地方因为是个数字，所以需要提前划分一下，划分成青年、中年和老年\n# 其中，年龄小于30为青年，30-60为中年，60以上为老年","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.385375Z","iopub.execute_input":"2022-06-21T03:09:23.385932Z","iopub.status.idle":"2022-06-21T03:09:23.392152Z","shell.execute_reply.started":"2022-06-21T03:09:23.385889Z","shell.execute_reply":"2022-06-21T03:09:23.391384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_split_user_set(split_standard,path):\n    csv_reader = csv.reader(open(path))\n    split_result_dict = {}\n    \n    for line in tqdm(csv_reader):\n        person_type = split_standard(line)\n        if person_type not in split_result_dict.keys():\n            split_result_dict[person_type] = set([line[0]])\n        else:\n            split_result_dict[person_type].add(line[0])\n    return split_result_dict\ndef club_member_status_split_func(line):\n    return line[3]\ndef fashion_news_frequency_split_func(line):\n    return line[4]\ndef age_split_func(line):\n    try:\n        age = int(line[5])\n    except:\n        age = 20\n    if age < 30:\n        return  \"y\"\n    if age < 60:\n        return \"m\"\n    else:\n        return \"o\"","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.393193Z","iopub.execute_input":"2022-06-21T03:09:23.393876Z","iopub.status.idle":"2022-06-21T03:09:23.403421Z","shell.execute_reply.started":"2022-06-21T03:09:23.393843Z","shell.execute_reply":"2022-06-21T03:09:23.402362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"club_split_result = get_split_user_set(club_member_status_split_func,\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:23.404519Z","iopub.execute_input":"2022-06-21T03:09:23.405264Z","iopub.status.idle":"2022-06-21T03:09:31.728051Z","shell.execute_reply.started":"2022-06-21T03:09:23.405231Z","shell.execute_reply":"2022-06-21T03:09:31.726871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 根据这一个结果反查这群人购买过的物品的集合，然后给存起来\ndef get_user_items_dict(path):\n    csv_reader = csv.reader(open(path))\n    user_items_dict = {}\n    for line in tqdm(csv_reader):\n        if line[0] not in user_items_dict.keys():\n            user_items_dict[line[1]] = set([line[2]])\n        else:\n            user_items_dict[line[1]].add(line[2])\n    return user_items_dict\nuser_items_dict = get_user_items_dict(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:09:31.729491Z","iopub.execute_input":"2022-06-21T03:09:31.729851Z","iopub.status.idle":"2022-06-21T03:13:10.105488Z","shell.execute_reply.started":"2022-06-21T03:09:31.729820Z","shell.execute_reply":"2022-06-21T03:13:10.104603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./person\")\n    os.mkdir(\"./person/club\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:10.111317Z","iopub.execute_input":"2022-06-21T03:13:10.112158Z","iopub.status.idle":"2022-06-21T03:13:10.118146Z","shell.execute_reply.started":"2022-06-21T03:13:10.112105Z","shell.execute_reply":"2022-06-21T03:13:10.117180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_person(origin_articles,user_items_dict,split_diction,file_path):\n    csv_writers = []\n    key_list = list(split_diction.keys())\n    for key in key_list:\n        csv_writers.append(csv.writer(open(file_path + \"/%s.csv\" % key,\"w\")))\n    article_set_for_each_type_dict = {}\n    for key in key_list:\n        person_set = split_diction[key]\n        article_set_for_each_type_dict[key] = list()\n        for person in person_set:\n            if person in user_items_dict.keys():\n                \n                article_set_for_each_type_dict[key] += list(user_items_dict[person])\n    for key in key_list:\n        article_set_for_each_type_dict[key] = set(article_set_for_each_type_dict[key])\n        print(len(article_set_for_each_type_dict[key]))\n    key_to_id = {}\n    for i,key in  enumerate(key_list):\n        key_to_id[key] = i\n    \n    csv_reader = csv.reader(open(origin_articles))\n\n    for line in tqdm(csv_reader):\n        for key in key_list:\n            if line[0] in article_set_for_each_type_dict[key]:\n                csv_writers[key_to_id[key]].writerow(line)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:10.119384Z","iopub.execute_input":"2022-06-21T03:13:10.119879Z","iopub.status.idle":"2022-06-21T03:13:10.135360Z","shell.execute_reply.started":"2022-06-21T03:13:10.119844Z","shell.execute_reply":"2022-06-21T03:13:10.134434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(club_split_result.keys())\nclub_split_result.pop(\"club_member_status\")\nclub_split_result.pop(\"LEFT CLUB\")\nclub_split_result.pop(\"\")\nprint(club_split_result.keys())","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:10.136653Z","iopub.execute_input":"2022-06-21T03:13:10.137022Z","iopub.status.idle":"2022-06-21T03:13:10.151034Z","shell.execute_reply.started":"2022-06-21T03:13:10.136978Z","shell.execute_reply":"2022-06-21T03:13:10.150041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(user_items_dict.keys())\nprint(\"197a52d35209d799a9a1670a35868276a30b9dbcffb2202209691228aa1e8339\" in user_items_dict.keys())","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:10.152541Z","iopub.execute_input":"2022-06-21T03:13:10.153263Z","iopub.status.idle":"2022-06-21T03:13:10.160940Z","shell.execute_reply.started":"2022-06-21T03:13:10.153226Z","shell.execute_reply":"2022-06-21T03:13:10.160224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_person(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"\n             ,user_items_dict,club_split_result,\n             \"./person/club\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:10.162170Z","iopub.execute_input":"2022-06-21T03:13:10.162946Z","iopub.status.idle":"2022-06-21T03:13:15.636602Z","shell.execute_reply.started":"2022-06-21T03:13:10.162913Z","shell.execute_reply":"2022-06-21T03:13:15.635331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fashion_split_result = get_split_user_set(fashion_news_frequency_split_func,\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:15.638492Z","iopub.execute_input":"2022-06-21T03:13:15.639012Z","iopub.status.idle":"2022-06-21T03:13:23.648162Z","shell.execute_reply.started":"2022-06-21T03:13:15.638964Z","shell.execute_reply":"2022-06-21T03:13:23.646999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fashion_split_result.keys())\nfashion_split_result.pop(\"fashion_news_frequency\")\nfashion_split_result.pop(\"\")\nfashion_split_result.pop(\"NONE\")\nfashion_split_result.pop(\"None\")\nprint(fashion_split_result.keys())","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:23.650186Z","iopub.execute_input":"2022-06-21T03:13:23.650680Z","iopub.status.idle":"2022-06-21T03:13:23.751770Z","shell.execute_reply.started":"2022-06-21T03:13:23.650632Z","shell.execute_reply":"2022-06-21T03:13:23.750595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./person/fashion\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:23.753187Z","iopub.execute_input":"2022-06-21T03:13:23.753522Z","iopub.status.idle":"2022-06-21T03:13:23.761214Z","shell.execute_reply.started":"2022-06-21T03:13:23.753490Z","shell.execute_reply":"2022-06-21T03:13:23.760228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_person(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"\n             ,user_items_dict,fashion_split_result,\n             \"./person/fashion\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:23.762129Z","iopub.execute_input":"2022-06-21T03:13:23.762422Z","iopub.status.idle":"2022-06-21T03:13:26.823840Z","shell.execute_reply.started":"2022-06-21T03:13:23.762393Z","shell.execute_reply":"2022-06-21T03:13:26.822649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"age_split_result = get_split_user_set(age_split_func,\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:26.825449Z","iopub.execute_input":"2022-06-21T03:13:26.826137Z","iopub.status.idle":"2022-06-21T03:13:35.470790Z","shell.execute_reply.started":"2022-06-21T03:13:26.826007Z","shell.execute_reply":"2022-06-21T03:13:35.470031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./person/age\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:35.471880Z","iopub.execute_input":"2022-06-21T03:13:35.472179Z","iopub.status.idle":"2022-06-21T03:13:35.479112Z","shell.execute_reply.started":"2022-06-21T03:13:35.472148Z","shell.execute_reply":"2022-06-21T03:13:35.478106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_person(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"\n             ,user_items_dict,age_split_result,\n             \"./person/age\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:35.480399Z","iopub.execute_input":"2022-06-21T03:13:35.480857Z","iopub.status.idle":"2022-06-21T03:13:41.759036Z","shell.execute_reply.started":"2022-06-21T03:13:35.480823Z","shell.execute_reply":"2022-06-21T03:13:41.758112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(\"./person/club\"))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:41.760680Z","iopub.execute_input":"2022-06-21T03:13:41.761018Z","iopub.status.idle":"2022-06-21T03:13:41.766911Z","shell.execute_reply.started":"2022-06-21T03:13:41.760986Z","shell.execute_reply":"2022-06-21T03:13:41.765943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"csv_reader = csv.reader(open(\"./person/club/ACTIVE.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:41.768690Z","iopub.execute_input":"2022-06-21T03:13:41.769617Z","iopub.status.idle":"2022-06-21T03:13:41.776148Z","shell.execute_reply.started":"2022-06-21T03:13:41.769536Z","shell.execute_reply":"2022-06-21T03:13:41.775113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_line = []","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:41.777479Z","iopub.execute_input":"2022-06-21T03:13:41.777842Z","iopub.status.idle":"2022-06-21T03:13:42.024772Z","shell.execute_reply.started":"2022-06-21T03:13:41.777807Z","shell.execute_reply":"2022-06-21T03:13:42.023505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for line in csv_reader:\n    temp_line.append(line)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:42.026068Z","iopub.execute_input":"2022-06-21T03:13:42.026426Z","iopub.status.idle":"2022-06-21T03:13:43.235303Z","shell.execute_reply.started":"2022-06-21T03:13:42.026385Z","shell.execute_reply":"2022-06-21T03:13:43.234346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(temp_line))","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:43.236501Z","iopub.execute_input":"2022-06-21T03:13:43.236922Z","iopub.status.idle":"2022-06-21T03:13:43.243512Z","shell.execute_reply.started":"2022-06-21T03:13:43.236887Z","shell.execute_reply":"2022-06-21T03:13:43.242590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.mkdir(\"./figures/person_comparation\")\n    os.mkdir(\"./figures/person_comparation/club\")\n    os.mkdir(\"./figures/person_comparation/fashion\")\n    os.mkdir(\"./figures/person_comparation/age\")\nexcept:\n    print(\"exist\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:43.245468Z","iopub.execute_input":"2022-06-21T03:13:43.245961Z","iopub.status.idle":"2022-06-21T03:13:43.252367Z","shell.execute_reply.started":"2022-06-21T03:13:43.245906Z","shell.execute_reply":"2022-06-21T03:13:43.251639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"person_classification_type = [\"fashion\",\"club\",\"age\"]\narticle_classification_type = [\"product_type_name\",\"colour_group_name\",\"index_group_name\",\"product_group_name\"]\nfor p_type in person_classification_type:\n    for attri in os.listdir(\"./person/%s\"%p_type):\n        for a_type in article_classification_type:\n            print(\"./figures/person_comparation/%s/%s-%s.jpg\"%(p_type,attri.replace(\".csv\",\"\"),a_type))\n            create_bar(\"./person/%s/%s\"%(p_type,attri),\n               article_attri_index_dict,a_type,\n               \"./figures/person_comparation/%s/%s-%s.jpg\"%(p_type,attri.replace(\".csv\",\"\"),a_type),\n               \"distribution of %s\"%a_type,\n               k=5)","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:13:43.253957Z","iopub.execute_input":"2022-06-21T03:13:43.254440Z","iopub.status.idle":"2022-06-21T03:16:10.465193Z","shell.execute_reply.started":"2022-06-21T03:13:43.254407Z","shell.execute_reply":"2022-06-21T03:16:10.464188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:16:10.466627Z","iopub.execute_input":"2022-06-21T03:16:10.466935Z","iopub.status.idle":"2022-06-21T03:16:11.375583Z","shell.execute_reply.started":"2022-06-21T03:16:10.466903Z","shell.execute_reply":"2022-06-21T03:16:11.374159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip figures.zip figures","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:16:11.378130Z","iopub.execute_input":"2022-06-21T03:16:11.379371Z","iopub.status.idle":"2022-06-21T03:16:12.261337Z","shell.execute_reply.started":"2022-06-21T03:16:11.379305Z","shell.execute_reply":"2022-06-21T03:16:12.259902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport zipfile\n\n\n# 压缩\ndef make_zip(source_dir, output_filename):\n    zipf = zipfile.ZipFile(output_filename, 'w')\n    pre_len = len(os.path.dirname(source_dir))\n    for parent, dirnames, filenames in os.walk(source_dir):\n        for filename in filenames:\n            print(filename)\n            pathfile = os.path.join(parent, filename)\n            arcname = pathfile[pre_len:].strip(os.path.sep)  # 相对路径\n            zipf.write(pathfile, arcname)\n        print()\n    zipf.close()\n","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:16:12.263177Z","iopub.execute_input":"2022-06-21T03:16:12.263534Z","iopub.status.idle":"2022-06-21T03:16:12.271590Z","shell.execute_reply.started":"2022-06-21T03:16:12.263499Z","shell.execute_reply":"2022-06-21T03:16:12.270891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_zip(\"./figures\",\"./all-figures.zip\")","metadata":{"execution":{"iopub.status.busy":"2022-06-21T03:31:47.226466Z","iopub.execute_input":"2022-06-21T03:31:47.227907Z","iopub.status.idle":"2022-06-21T03:31:48.115017Z","shell.execute_reply.started":"2022-06-21T03:31:47.227810Z","shell.execute_reply":"2022-06-21T03:31:48.113747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}