{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-09T22:56:27.663184Z","iopub.execute_input":"2022-03-09T22:56:27.663849Z","iopub.status.idle":"2022-03-09T22:56:27.691401Z","shell.execute_reply.started":"2022-03-09T22:56:27.663740Z","shell.execute_reply":"2022-03-09T22:56:27.690634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ndf = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', dtype={'article_id': str})\ndf_sub = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\ndf['sales_channel_id'] = df['sales_channel_id'].astype('int8')\nmeta_art = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv', dtype={'article_id': str})\nmeta_art.drop([\n               'graphical_appearance_no','product_type_no','colour_group_code',\\\n               'index_code','index_group_no','product_code',\\\n               'perceived_colour_value_id','perceived_colour_master_id',\\\n               'department_no','section_no','garment_group_no'],\\\n              axis=1,inplace=True)\nmeta_art.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:56:27.693897Z","iopub.execute_input":"2022-03-09T22:56:27.694521Z","iopub.status.idle":"2022-03-09T22:57:52.964967Z","shell.execute_reply.started":"2022-03-09T22:56:27.694475Z","shell.execute_reply":"2022-03-09T22:57:52.963899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we create a sample of customers who ordered during the last week of data\n# The goal is to have enough data to avoid overfitting as much as possible\ncustomer_list = random.sample(set(df[df['t_dat']>'2020-09-14']['customer_id']),\\\n                              k=10000)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:57:52.966538Z","iopub.execute_input":"2022-03-09T22:57:52.966806Z","iopub.status.idle":"2022-03-09T22:57:57.459891Z","shell.execute_reply.started":"2022-03-09T22:57:52.966774Z","shell.execute_reply":"2022-03-09T22:57:57.458939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we create the dataframe combining transaction and articles by customer\ndf_info = df.merge(meta_art,how='left',on='article_id')\n# we create the df to estimate some data in the 2 weeks before the last one\ndf_2_weeks = df_info[(df_info['t_dat']>'2020-09-01')&(df_info['t_dat']<='2020-09-14')]\n# we create a sub dataframe restricted to our sample to speed up calcs\ndf_train = df_info[df_info['customer_id'].isin(customer_list)]\ndel df_info\n# we create the verification dataframe\ndf_temp_Y = df[df['customer_id'].isin(customer_list)]\ndf_temp_Y = df_temp_Y[df_temp_Y['t_dat'] > '2020-09-14']","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:57:57.463215Z","iopub.execute_input":"2022-03-09T22:57:57.463756Z","iopub.status.idle":"2022-03-09T22:58:51.018865Z","shell.execute_reply.started":"2022-03-09T22:57:57.463705Z","shell.execute_reply":"2022-03-09T22:58:51.017868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# here are my ideas of variables that could help reducing the size of the candidate articles\nsample_train_df = pd.DataFrame(index=customer_list)\nsample_train_df['customer_id']=sample_train_df.index\nsample_train_df['colour_bought'] = df_train.groupby('customer_id')\\\n['perceived_colour_master_name'].unique()\nsample_train_df['type_of_clothing'] = df_train.groupby('customer_id')\\\n['product_group_name'].unique()\nsample_train_df['appearance'] = df_train.groupby('customer_id')\\\n['graphical_appearance_name'].unique()\nsample_train_df['max_budget'] = df_train.groupby('customer_id')['price'].max()\nsample_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:58:51.020456Z","iopub.execute_input":"2022-03-09T22:58:51.020704Z","iopub.status.idle":"2022-03-09T22:58:53.817417Z","shell.execute_reply.started":"2022-03-09T22:58:51.020677Z","shell.execute_reply":"2022-03-09T22:58:53.816180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this function is meant to take a list of article and remove those that do not have\n# a colour previously bought by the customer\n# it returns the intersection of article bought during the last week and the newly created list\n# it also returns the number of different bought articles during the last week\n# it also returns the number of articles in the newly created list\ndef art_sel_colour(cust): \n    actually_bought = df_temp_Y[df_temp_Y['customer_id']==cust]['article_id']    \n    temp_list = wide_list_popular_item_df\n    # we remove articles with perceived colour never bought by the customer\n    temp_list = set(temp_list.index).intersection(set(meta_art_pop[meta_art_pop['perceived_colour_master_name'].isin(set(sample_train_df.loc[cust,'colour_bought']))]['article_id'])) \n    temp_list1 = temp_list.intersection(set(actually_bought)) \n    return(len(temp_list1),len(set(actually_bought)),len(temp_list))","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:58:53.818873Z","iopub.execute_input":"2022-03-09T22:58:53.819204Z","iopub.status.idle":"2022-03-09T22:58:53.827877Z","shell.execute_reply.started":"2022-03-09T22:58:53.819161Z","shell.execute_reply":"2022-03-09T22:58:53.826829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we define the list of most popular items with a certain threshold to\n# reach a certain percentage of sales\n# we try different thresholds in th_list and store the results in count1,2,3\nth_list = [0.05,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]\ncount1 = []\ncount2 = []\ncount3 = []\nfor th in th_list:\n  nb = (df_2_weeks['article_id'].value_counts(normalize=True).cumsum()<th).sum() \n  wide_list_popular_items = df_2_weeks['article_id'].value_counts(\n      normalize=True).head(nb).index.tolist()\n  wide_list_popular_item_df = pd.DataFrame(index=wide_list_popular_items)\n  # we calculate the estimated price of each of these items\n  # (sales can change price)\n  # we create a subdataframe restricted to the popular item defined by th\n  meta_art_pop = meta_art[meta_art['article_id'].isin(wide_list_popular_items)]\n  # we calculate c1 the number of articles bought and in our restricted\n  # list defined in article_selection for all customers we divide c1 by the \n  # number of purchases to define the maximal precision if we chose the\n  # articles in our restricted list we calculate c2 the number of articles\n  # in our restricted list on average we calculate c3 which is c1/c2 the\n  # required 'precision' to find the good articles in our list  \n  sample_train_df['possible_good_guess'] = sample_train_df['customer_id'].apply(\n      art_sel_colour)  \n  c1 = 0\n  c2 = 0  \n  c3 = 0\n  for i in sample_train_df['possible_good_guess']:    \n    c1 += i[0]\n    c2 += i[2]    \n    c3 += i[1]       \n  c2 = c2/len(sample_train_df['customer_id'])  \n  count1.append(c1/c3)\n  count2.append(c2)\n  count3.append((1-c1/c3)*len(df_temp_Y['article_id'])/(len(df_temp_Y['article_id'])-c2)) \n  # count1 is max precision that we can expect if the algorithm has 100% precision\n  # count2 is the number of articles per customer on average\n  # count3 is a fight between two terms (1-c1/len(df_temp_Y['customer_id'])) the loss of maximal potential precision,\n  # and (len(df_temp_Y['article_id'])-c2))/len(df_temp_Y['article_id']) the loss of candidates in percentage\n  # so we ideally want count3 to be as close to 0 as possible while also having c2 as low as possible\n  # the work here was done on only 2 variables popularity and perceived colour, but we can imagine chaining reductions","metadata":{"execution":{"iopub.status.busy":"2022-03-09T22:58:53.829518Z","iopub.execute_input":"2022-03-09T22:58:53.829921Z","iopub.status.idle":"2022-03-09T23:13:57.188896Z","shell.execute_reply.started":"2022-03-09T22:58:53.829887Z","shell.execute_reply":"2022-03-09T23:13:57.187496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.relplot(x=th_list,y=count1)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T23:13:57.191377Z","iopub.execute_input":"2022-03-09T23:13:57.191642Z","iopub.status.idle":"2022-03-09T23:13:57.591569Z","shell.execute_reply.started":"2022-03-09T23:13:57.191609Z","shell.execute_reply":"2022-03-09T23:13:57.590695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.relplot(x=th_list,y=count2)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T23:13:57.593182Z","iopub.execute_input":"2022-03-09T23:13:57.593391Z","iopub.status.idle":"2022-03-09T23:13:57.882779Z","shell.execute_reply.started":"2022-03-09T23:13:57.593366Z","shell.execute_reply":"2022-03-09T23:13:57.882220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.relplot(x=th_list,y=count3)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T23:13:57.885074Z","iopub.execute_input":"2022-03-09T23:13:57.886199Z","iopub.status.idle":"2022-03-09T23:13:58.198249Z","shell.execute_reply.started":"2022-03-09T23:13:57.886145Z","shell.execute_reply":"2022-03-09T23:13:58.197549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count1","metadata":{"execution":{"iopub.status.busy":"2022-03-10T00:20:58.211488Z","iopub.execute_input":"2022-03-10T00:20:58.211844Z","iopub.status.idle":"2022-03-10T00:20:58.218557Z","shell.execute_reply.started":"2022-03-10T00:20:58.211807Z","shell.execute_reply":"2022-03-10T00:20:58.217894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count2","metadata":{"execution":{"iopub.status.busy":"2022-03-10T00:21:04.532304Z","iopub.execute_input":"2022-03-10T00:21:04.533070Z","iopub.status.idle":"2022-03-10T00:21:04.539228Z","shell.execute_reply.started":"2022-03-10T00:21:04.533031Z","shell.execute_reply":"2022-03-10T00:21:04.538371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count3","metadata":{"execution":{"iopub.status.busy":"2022-03-10T00:21:12.195920Z","iopub.execute_input":"2022-03-10T00:21:12.196678Z","iopub.status.idle":"2022-03-10T00:21:12.204310Z","shell.execute_reply.started":"2022-03-10T00:21:12.196620Z","shell.execute_reply":"2022-03-10T00:21:12.203176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}