{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# H&M Personalized Fashion Recommendations\n\n\n\n##### Goal: To predict what articles each customer will purchase in the 7-day period immediately after the training data ends. \n\n\n\n\n##### Files:\n* articles.csv - detailed metadata for each article_id available for purchase
\n* customers.csv - metadata for each customer_id in dataset
\n* sample_submission.csv - a sample submission used for validation
\n* transactions_train.csv - the training data, consisting of the purchases each customer for each date, as well as additional information
","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nimport gc","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-05-10T18:01:12.729469Z","iopub.execute_input":"2022-05-10T18:01:12.729792Z","iopub.status.idle":"2022-05-10T18:01:12.734997Z","shell.execute_reply.started":"2022-05-10T18:01:12.729761Z","shell.execute_reply":"2022-05-10T18:01:12.734361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the files\ntransactions_train = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")\nsubmissions = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\")\narticles = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:01:12.942274Z","iopub.execute_input":"2022-05-10T18:01:12.942896Z","iopub.status.idle":"2022-05-10T18:02:29.874606Z","shell.execute_reply.started":"2022-05-10T18:01:12.942817Z","shell.execute_reply":"2022-05-10T18:02:29.873617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Apache Parquet** is a free and open-source column-oriented data storage format of the Apache Hadoop ecosystem. Parquet operates well with complex data in large volumes. It is known for its both performant data compression and its ability to handle a wide variety of encoding types. ","metadata":{}},{"cell_type":"code","source":"# Converting to Parquet to shrink size and save memory \narticles.to_parquet('articles.parquet')\ncustomers.to_parquet('customers.parquet')\ntransactions_train.to_parquet('transactions_train.parquet')\nsubmissions.to_parquet('submission.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:02:29.880021Z","iopub.execute_input":"2022-05-10T18:02:29.881608Z","iopub.status.idle":"2022-05-10T18:02:48.219647Z","shell.execute_reply.started":"2022-05-10T18:02:29.881561Z","shell.execute_reply":"2022-05-10T18:02:48.218646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cross Validation Scheme\nThe first step is to build a reliable local validation scheme. Then we use our local validation score to evaluate experiment ideas and/or tune hyperparameters.\n\n**Train Data**\n\nThe last day in the transaction dataframe is 2020-09-22. The public LB contains 1 week of transactions after this date. Therefore to create a local validation that mimics Kaggle's train test relationship, we can train on all transactions before 2020-9-15. And validate on the last week in train data.","metadata":{}},{"cell_type":"code","source":"# Creating a local cross validation scheme\ntrain = pd.read_parquet('transactions_train.parquet')\ntrain.t_dat = pd.to_datetime( train.t_dat )\ntrain = train.loc[ train.t_dat <= pd.to_datetime('2020-09-15') ]","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:02:48.221368Z","iopub.execute_input":"2022-05-10T18:02:48.222006Z","iopub.status.idle":"2022-05-10T18:03:11.442989Z","shell.execute_reply.started":"2022-05-10T18:02:48.221941Z","shell.execute_reply":"2022-05-10T18:03:11.442052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Validation Data**\n\nThe code below will create a dataframe with only the customers who made purchases during the last week of train (which are the only ones that affect competition metric).\nIt formats the predictions as strings like sample_submission.csv","metadata":{}},{"cell_type":"code","source":"valid = pd.read_parquet('transactions_train.parquet')\nvalid.t_dat = pd.to_datetime( valid.t_dat )\ntest = valid.loc[ valid.t_dat >= pd.to_datetime('2020-09-16') ]\ntest = test.groupby('customer_id').article_id.apply(list).reset_index()\ntest = test.rename({'article_id':'prediction'},axis=1)\ntest['prediction'] = test.prediction.apply(lambda x: ' '.join(['0'+str(k) for k in x]))","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:11.447154Z","iopub.execute_input":"2022-05-10T18:03:11.448198Z","iopub.status.idle":"2022-05-10T18:03:36.525315Z","shell.execute_reply.started":"2022-05-10T18:03:11.448136Z","shell.execute_reply":"2022-05-10T18:03:36.524502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_parquet(\"./articles.parquet\")\ncustomers = pd.read_parquet(\"./customers.parquet\")\ntransactions = pd.read_parquet(\"./transactions_train.parquet\")\nsubmission = pd.read_parquet(\"./submission.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:36.527875Z","iopub.execute_input":"2022-05-10T18:03:36.529000Z","iopub.status.idle":"2022-05-10T18:03:56.220427Z","shell.execute_reply.started":"2022-05-10T18:03:36.528958Z","shell.execute_reply":"2022-05-10T18:03:56.219514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Understanding the data**","metadata":{}},{"cell_type":"markdown","source":"#### **Articles.csv**\n##### The Articles file contains information about all the products that are sold. The article_id is the primary key. There are 1,05,542 products present in the data. There are 24 columns which contain information about each product.\n* **Primary key**: \n * 'article_id'
\n
\n* **Basic information about the Product and it's Category**: \n * product_code, prod_name, product_type_no, product_type_name, product_group_name
\n
\n* **Visual Classification of the products**:\n * Graphical Appearance:
\n 'graphical_appearance_no','graphical_appearance_name'
\n \n * Colour:
\n 'colour_group_code', 'colour_group_name','perceived_colour_value_id', 'perceived_colour_value_name', 'perceived_colour_master_id', 'perceived_colour_master_name'\n\n* **Classification of Products by Department:** \n * Department:
\n 'department_no', 'department_name',
\n\n * Shop Index :
\n 'index_code', 'index_name', 'index_group_no', 'index_group_name'\n\n * Section :
\n 'section_no', 'section_name'
\n \n * Garment group:
\n 'garment_group_no', 'garment_group_name', 'detail_desc'
\n","metadata":{}},{"cell_type":"code","source":"print(articles.shape)\narticles.tail()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.221731Z","iopub.execute_input":"2022-05-10T18:03:56.221979Z","iopub.status.idle":"2022-05-10T18:03:56.252681Z","shell.execute_reply.started":"2022-05-10T18:03:56.221950Z","shell.execute_reply":"2022-05-10T18:03:56.251801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Customers.csv**\n##### The Customers file contains information about all the products that are sold. The customer_id is the primary key. There are 13,71,980 customers present in the data. There are 6 columns which contain information about each customer.\n* **Primary key:** \n * 'customer_id'
\n
\n* **Information about whether the customer has subscribed to the Fashion News Updates: **\n * FN, Active, club_member_status, fashion_news_frequency
\n
\n* **Age**
\n\n* **Postal_code **","metadata":{}},{"cell_type":"code","source":"print(customers.shape)\ncustomers.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.254389Z","iopub.execute_input":"2022-05-10T18:03:56.254695Z","iopub.status.idle":"2022-05-10T18:03:56.272776Z","shell.execute_reply.started":"2022-05-10T18:03:56.254653Z","shell.execute_reply":"2022-05-10T18:03:56.271976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Transactions.csv**\n##### The Transactions file contains information about all past transactions of customers from the years 2020 to 2022. There are 3,17,88,324 transactions present in the data. There are 4 columns which contain information about each transaction.\n##### This has `customer_id` and `article_id`, which are foreign keys for the articles and customer tables.\nBeside this, transaction also contains `sales_channel_id`.\n\n* **Transaction Date:** \n * 't_dat'
\n
\n* **Customer and Product information:**\n * customer_id, article_id, price
\n
\n* **Sales Channel (Online/ Offline)**
\n * sales_channel_id","metadata":{}},{"cell_type":"code","source":"print(transactions.shape)\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.273995Z","iopub.execute_input":"2022-05-10T18:03:56.274220Z","iopub.status.idle":"2022-05-10T18:03:56.289684Z","shell.execute_reply.started":"2022-05-10T18:03:56.274193Z","shell.execute_reply":"2022-05-10T18:03:56.288853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining Helper Functions\n\ndef missing_data(data):\n total = data.isnull().sum().sort_values(ascending = False)\n percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)\n return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n\ndef unique_values(data):\n total = data.count()\n tt = pd.DataFrame(total)\n tt.columns = ['Total']\n uniques = []\n for col in data.columns:\n unique = data[col].nunique()\n uniques.append(unique)\n tt['Uniques'] = uniques\n return tt ","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.290861Z","iopub.execute_input":"2022-05-10T18:03:56.291452Z","iopub.status.idle":"2022-05-10T18:03:56.299565Z","shell.execute_reply.started":"2022-05-10T18:03:56.291419Z","shell.execute_reply":"2022-05-10T18:03:56.298701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Check for Missing Values","metadata":{}},{"cell_type":"code","source":"# 416 products have missing descriptions (Only 0.4% of the data)\nmissing_data(articles)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.303660Z","iopub.execute_input":"2022-05-10T18:03:56.304025Z","iopub.status.idle":"2022-05-10T18:03:56.824574Z","shell.execute_reply.started":"2022-05-10T18:03:56.303983Z","shell.execute_reply":"2022-05-10T18:03:56.823713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only customer id and postal code are completely filled. Age, fashion news frequency have arounfd 1% missing data, FN has 65% missing and Active has 66% missing data.\nmissing_data(customers)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:56.825964Z","iopub.execute_input":"2022-05-10T18:03:56.826273Z","iopub.status.idle":"2022-05-10T18:03:58.921553Z","shell.execute_reply.started":"2022-05-10T18:03:56.826233Z","shell.execute_reply":"2022-05-10T18:03:58.920700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No missing data from transactions data\nmissing_data(transactions)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:03:58.923085Z","iopub.execute_input":"2022-05-10T18:03:58.923539Z","iopub.status.idle":"2022-05-10T18:04:22.700354Z","shell.execute_reply.started":"2022-05-10T18:03:58.923497Z","shell.execute_reply":"2022-05-10T18:04:22.699325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Unique Values","metadata":{}},{"cell_type":"code","source":"unique_values(articles)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:22.701752Z","iopub.execute_input":"2022-05-10T18:04:22.702189Z","iopub.status.idle":"2022-05-10T18:04:23.049832Z","shell.execute_reply.started":"2022-05-10T18:04:22.702134Z","shell.execute_reply":"2022-05-10T18:04:23.049183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that features for which we expect to have the same number of unique value, like:\n* product_type_no and product_type_name, \n* departmant_no and department_name, \n* section_no and section_name \nhave different number of unique values, which might means that we might have categories with same name.\nOthers, like:\n* index_code and index_name,\n* garment_group_no and garment_group_name\nhave the same number of unique values.","metadata":{}},{"cell_type":"code","source":"unique_values(customers)","metadata":{"_kg_hide-input":true,"papermill":{"duration":2.284684,"end_time":"2022-02-13T21:00:23.914440","exception":false,"start_time":"2022-02-13T21:00:21.629756","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-10T18:04:23.050896Z","iopub.execute_input":"2022-05-10T18:04:23.051545Z","iopub.status.idle":"2022-05-10T18:04:25.392053Z","shell.execute_reply.started":"2022-05-10T18:04:23.051510Z","shell.execute_reply":"2022-05-10T18:04:25.391394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values(transactions)","metadata":{"_kg_hide-input":true,"papermill":{"duration":13.884573,"end_time":"2022-02-13T21:00:37.942480","exception":false,"start_time":"2022-02-13T21:00:24.057907","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-10T18:04:25.393380Z","iopub.execute_input":"2022-05-10T18:04:25.393677Z","iopub.status.idle":"2022-05-10T18:04:46.212169Z","shell.execute_reply.started":"2022-05-10T18:04:25.393635Z","shell.execute_reply":"2022-05-10T18:04:46.211381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that not all the customers (in the customer data) have transactions in the transaction data. As well as, not all articles are represented in this data. It is interesting that the number of different prices is quite small, out of 31.7M transactions, and for 1.3M customers, buying 104K different articles. Same for the dates, there are only 734 different dates. Let's check some stats here.","metadata":{"papermill":{"duration":0.152288,"end_time":"2022-02-13T21:00:38.245762","exception":false,"start_time":"2022-02-13T21:00:38.093474","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"Percentage of articles present in the transactions data: {round(transactions.article_id.nunique()/articles.article_id.nunique()*100, 2)}%\")\nprint(f\"Percentage of customers present in the transactions data: {round(transactions.customer_id.nunique()*100/customers.customer_id.nunique(), 2)}%\")","metadata":{"papermill":{"duration":0.103351,"end_time":"2022-02-13T21:00:38.463292","exception":false,"start_time":"2022-02-13T21:00:38.359941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-10T18:04:46.213346Z","iopub.execute_input":"2022-05-10T18:04:46.213579Z","iopub.status.idle":"2022-05-10T18:04:56.916678Z","shell.execute_reply.started":"2022-05-10T18:04:46.213551Z","shell.execute_reply":"2022-05-10T18:04:56.915646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Preprocessing**","metadata":{}},{"cell_type":"code","source":"# There is a mistake in data entry where NONE and None have been entered differently\nprint(customers.fashion_news_frequency.value_counts())\ncustomers.loc[customers[\"fashion_news_frequency\"] == \"NONE\",\"fashion_news_frequency\"] = \"None\"\ncustomers.fashion_news_frequency.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:56.918048Z","iopub.execute_input":"2022-05-10T18:04:56.918254Z","iopub.status.idle":"2022-05-10T18:04:57.499322Z","shell.execute_reply.started":"2022-05-10T18:04:56.918228Z","shell.execute_reply":"2022-05-10T18:04:57.498388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **EDA**","metadata":{}},{"cell_type":"code","source":"# Most popular products\narticles.prod_name.value_counts()[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:57.500460Z","iopub.execute_input":"2022-05-10T18:04:57.500684Z","iopub.status.idle":"2022-05-10T18:04:57.548360Z","shell.execute_reply.started":"2022-05-10T18:04:57.500659Z","shell.execute_reply":"2022-05-10T18:04:57.547761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Product Types per Product Group\nprod_grps = articles.groupby([\"product_group_name\"])[\"product_type_name\"].nunique()\ndf = pd.DataFrame({'Product Group': prod_grps.index,\n 'Product Types': prod_grps.values\n })\ndf = df.sort_values(['Product Types'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title('Number of Product Types per Product Group')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x=\"Product Types\", y = 'Product Group', data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:57.549246Z","iopub.execute_input":"2022-05-10T18:04:57.549445Z","iopub.status.idle":"2022-05-10T18:04:57.945220Z","shell.execute_reply.started":"2022-05-10T18:04:57.549420Z","shell.execute_reply":"2022-05-10T18:04:57.944118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Product Type\nprod_types = articles.groupby([\"product_type_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Product Type': prod_types.index,\n 'Articles': prod_types.values\n })\ntotal_types = len(df['Product Type'].unique())\ndf = df.sort_values(['Articles'], ascending=False)[0:50]\nplt.figure(figsize = (20,15))\nplt.title(f'Number of Articles per Product Type (top 50 from total: {total_types})')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(y = 'Product Type', x=\"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:57.946694Z","iopub.execute_input":"2022-05-10T18:04:57.947051Z","iopub.status.idle":"2022-05-10T18:04:58.840600Z","shell.execute_reply.started":"2022-05-10T18:04:57.947004Z","shell.execute_reply":"2022-05-10T18:04:58.839631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Department\ntemp = articles.groupby([\"department_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Department Name': temp.index,\n 'Articles': temp.values\n })\ntotal_depts = len(df['Department Name'].unique())\ndf = df.sort_values(['Articles'], ascending=False).head(50)\nplt.figure(figsize = (16,15))\nplt.title(f'Number of Articles per each Department (top 50 from total: {total_depts})')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(y = 'Department Name', x=\"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:58.842382Z","iopub.execute_input":"2022-05-10T18:04:58.842731Z","iopub.status.idle":"2022-05-10T18:04:59.942398Z","shell.execute_reply.started":"2022-05-10T18:04:58.842686Z","shell.execute_reply":"2022-05-10T18:04:59.941757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Graphical Appearance Name\ntemp = articles.groupby([\"graphical_appearance_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Graphical Appearance Name': temp.index,\n 'Articles': temp.values\n })\ndf = df.sort_values(['Articles'], ascending=False).head(50)\nplt.figure(figsize = (16,15))\nplt.title(f'Number of Articles per each Graphical Appearance Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(y = 'Graphical Appearance Name', x = \"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:04:59.943708Z","iopub.execute_input":"2022-05-10T18:04:59.944613Z","iopub.status.idle":"2022-05-10T18:05:00.402393Z","shell.execute_reply.started":"2022-05-10T18:04:59.944564Z","shell.execute_reply":"2022-05-10T18:05:00.401461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Index Name\ntemp = articles.groupby([\"index_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Index Name': temp.index,\n 'Articles': temp.values\n })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (8,6))\nplt.title(f'Number of Articles per each Index Name')\nsns.set_color_codes(\"bright\")\ns = sns.barplot(y = 'Index Name', x=\"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:00.403817Z","iopub.execute_input":"2022-05-10T18:05:00.404104Z","iopub.status.idle":"2022-05-10T18:05:00.665863Z","shell.execute_reply.started":"2022-05-10T18:05:00.404070Z","shell.execute_reply":"2022-05-10T18:05:00.665031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Index Group Name\ntemp = articles.groupby([\"colour_group_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Colour Group Name': temp.index,\n 'Articles': temp.values\n })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (12,16))\nplt.title(f'Number of Articles per each Colour Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(y = 'Colour Group Name', x=\"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:00.667191Z","iopub.execute_input":"2022-05-10T18:05:00.667803Z","iopub.status.idle":"2022-05-10T18:05:01.533272Z","shell.execute_reply.started":"2022-05-10T18:05:00.667767Z","shell.execute_reply":"2022-05-10T18:05:01.532407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Articles per Perceived Colour Name\ntemp = articles.groupby([\"perceived_colour_value_name\"])[\"article_id\"].nunique()\ndf = pd.DataFrame({'Perceived Colour Group Name': temp.index,\n 'Articles': temp.values\n })\ndf = df.sort_values(['Articles'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Articles per each Perceived Colour Group Name')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(y ='Perceived Colour Group Name', x=\"Articles\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:01.534677Z","iopub.execute_input":"2022-05-10T18:05:01.534952Z","iopub.status.idle":"2022-05-10T18:05:01.793722Z","shell.execute_reply.started":"2022-05-10T18:05:01.534916Z","shell.execute_reply":"2022-05-10T18:05:01.792767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distibution of Customers across Age Group\ntemp = customers.groupby([\"age\"])[\"customer_id\"].count()\ndf = pd.DataFrame({'Age': temp.index,\n 'Customers': temp.values\n })\ndf = df.sort_values(['Age'], ascending=False)\nplt.figure(figsize = (20,6))\nplt.title(f'Number of Customers per each Age')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Age', y=\"Customers\", data=df)\ns.set_xticklabels(s.get_xticklabels(),rotation=90)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:01.795265Z","iopub.execute_input":"2022-05-10T18:05:01.795857Z","iopub.status.idle":"2022-05-10T18:05:03.122430Z","shell.execute_reply.started":"2022-05-10T18:05:01.795809Z","shell.execute_reply":"2022-05-10T18:05:03.121594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Number of customers and thier Fashion News Frequency\nprint(customers.fashion_news_frequency.value_counts())\ntemp = customers.groupby([\"fashion_news_frequency\"])[\"customer_id\"].count()\ndf = pd.DataFrame({'Fashion News Frequency': temp.index,\n 'Customers': temp.values\n }) \ndf = df.sort_values(['Customers'], ascending=False)\nplt.figure(figsize = (6,6))\nplt.title(f'Number of Customers per each Fashion News Frequency')\nsns.set_color_codes(\"pastel\")\ns = sns.barplot(x = 'Fashion News Frequency', y=\"Customers\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:03.123814Z","iopub.execute_input":"2022-05-10T18:05:03.124091Z","iopub.status.idle":"2022-05-10T18:05:03.765718Z","shell.execute_reply.started":"2022-05-10T18:05:03.124061Z","shell.execute_reply":"2022-05-10T18:05:03.765096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Daily Sales (Transactions)\ndf = transactions.groupby([\"t_dat\"])[\"article_id\"].count().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Transactions\"]\nfig, ax = plt.subplots(1, 1, figsize=(20,6))\nplt.plot(df[\"Date\"], df[\"Transactions\"], color=\"Darkblue\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Transactions\")\nplt.title(f\"Daily Sales (Transactions)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:03.769091Z","iopub.execute_input":"2022-05-10T18:05:03.769493Z","iopub.status.idle":"2022-05-10T18:05:07.328578Z","shell.execute_reply.started":"2022-05-10T18:05:03.769463Z","shell.execute_reply":"2022-05-10T18:05:07.327956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of the transactions over time\ndf = transactions.groupby([\"t_dat\", \"sales_channel_id\"])[\"article_id\"].count().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Sales Channel Id\", \"Transactions\"]\nfig, ax = plt.subplots(1, 1, figsize=(16,6))\ng1 = ax.plot(df.loc[df[\"Sales Channel Id\"]==1, \"Date\"], df.loc[df[\"Sales Channel Id\"]==1, \"Transactions\"], label=\"Sales Channel 1\", color=\"Darkblue\")\ng2 = ax.plot(df.loc[df[\"Sales Channel Id\"]==2, \"Date\"], df.loc[df[\"Sales Channel Id\"]==2, \"Transactions\"], label=\"Sales Channel 2\", color=\"Magenta\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Transactions\")\nax.legend()\nplt.title(f\"Transactions per day, grouped by Sales Channel\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:07.329849Z","iopub.execute_input":"2022-05-10T18:05:07.330301Z","iopub.status.idle":"2022-05-10T18:05:12.331014Z","shell.execute_reply.started":"2022-05-10T18:05:07.330269Z","shell.execute_reply":"2022-05-10T18:05:12.330318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of Number of Unique articles sold each day\ndf = transactions.groupby([\"t_dat\", \"sales_channel_id\"])[\"article_id\"].nunique().reset_index()\ndf[\"t_dat\"] = df[\"t_dat\"].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d'))\ndf.columns = [\"Date\", \"Sales Channel Id\", \"Unique Articles\"]\nfig, ax = plt.subplots(1, 1, figsize=(16,6))\ng1 = ax.plot(df.loc[df[\"Sales Channel Id\"]==1, \"Date\"], df.loc[df[\"Sales Channel Id\"]==1, \"Unique Articles\"], label=\"Sales Channel 1\", color=\"Blue\")\ng2 = ax.plot(df.loc[df[\"Sales Channel Id\"]==2, \"Date\"], df.loc[df[\"Sales Channel Id\"]==2, \"Unique Articles\"], label=\"Sales Channel 2\", color=\"Green\")\nplt.xlabel(\"Date\")\nplt.ylabel(\"Unique Articles / Day\")\nax.legend()\nplt.title(f\"Unique articles per day, grouped by Sales Channel\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:12.332361Z","iopub.execute_input":"2022-05-10T18:05:12.332776Z","iopub.status.idle":"2022-05-10T18:05:29.321816Z","shell.execute_reply.started":"2022-05-10T18:05:12.332744Z","shell.execute_reply":"2022-05-10T18:05:29.320938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Modelling**","metadata":{}},{"cell_type":"code","source":"# Defining the scoring metrics\n\ndef apk(actual, predicted, k=10):\n \"\"\"\n Computes the average precision at k.\n This function computes the average prescision at k between two lists of\n items.\n Parameters\n ----------\n actual : list\n A list of elements that are to be predicted (order doesn't matter)\n predicted : list\n A list of predicted elements (order does matter)\n k : int, optional\n The maximum number of predicted elements\n Returns\n -------\n score : double\n The average precision at k over the input lists\n \"\"\"\n if len(predicted)>k:\n predicted = predicted[:k]\n\n score = 0.0\n num_hits = 0.0\n\n for i,p in enumerate(predicted):\n if p in actual and p not in predicted[:i]:\n num_hits += 1.0\n score += num_hits / (i+1.0)\n\n if not actual:\n return 0.0\n\n return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n \"\"\"\n Computes the mean average precision at k.\n This function computes the mean average prescision at k between two lists\n of lists of items.\n Parameters\n ----------\n actual : list\n A list of lists of elements that are to be predicted \n (order doesn't matter in the lists)\n predicted : list\n A list of lists of predicted elements\n (order matters in the lists)\n k : int, optional\n The maximum number of predicted elements\n Returns\n -------\n score : double\n The mean average precision at k over the input lists\n \"\"\"\n return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:29.323136Z","iopub.execute_input":"2022-05-10T18:05:29.323377Z","iopub.status.idle":"2022-05-10T18:05:29.332536Z","shell.execute_reply.started":"2022-05-10T18:05:29.323347Z","shell.execute_reply":"2022-05-10T18:05:29.331572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Top Ranking Recently Bought Items based Model:**\n* If there are articles for a certain client, pick the most recent buys\n* If there are not articles for a certain client, just pick the most frequently bought articles.","metadata":{}},{"cell_type":"code","source":"transactions2 = train.sort_values([\"customer_id\", \"t_dat\"], ascending=False)\n# Capturing most frequently bought articles\nlast_date = transactions2.t_dat.max()\nmost_frequent_articles = list(transactions2.loc[transactions2.t_dat==last_date].article_id.value_counts()[0:12].index)\nart_list = []\nfor art in most_frequent_articles:\n art = \"0\"+str(art)\n art_list.append(art)\nart_str = \" \".join(art_list)\nprint(\"Frequent articles bought recently: \", art_str)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:29.333872Z","iopub.execute_input":"2022-05-10T18:05:29.334387Z","iopub.status.idle":"2022-05-10T18:05:53.334904Z","shell.execute_reply.started":"2022-05-10T18:05:29.334352Z","shell.execute_reply":"2022-05-10T18:05:53.333931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df = transactions2.groupby([\"customer_id\"])[\"article_id\"].agg(lambda x: str(x.values[0:12])[1:-1]).reset_index()\n\ndef padding_articles(x):\n if x:\n xl = x.split()\n x = []\n for xi in xl:\n x.append(\"0\"+xi)\n dimm_x = len(x)\n if dimm_x < 12:\n x.extend(art_list[:12-dimm_x])\n return(\" \".join(x))\n\nagg_df[\"article_id\"] = agg_df[\"article_id\"].apply(lambda x: padding_articles(x))\n\nrecommendations = agg_df.merge(submission[[\"customer_id\"]], how=\"right\")\nrecommendations.columns = [\"customer_id\", \"prediction\"]\nrecommendations.loc[recommendations.prediction.isna(), [\"prediction\"]] = art_str\nrecommendations.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:05:53.336454Z","iopub.execute_input":"2022-05-10T18:05:53.337100Z","iopub.status.idle":"2022-05-10T18:07:40.193573Z","shell.execute_reply.started":"2022-05-10T18:05:53.337056Z","shell.execute_reply":"2022-05-10T18:07:40.192951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = recommendations\nsub = recommendations.set_index('customer_id').loc[test.customer_id].reset_index()\nround(mapk(test.prediction.str.split(), sub.prediction.str.split(), k=12), 3)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T18:09:40.453276Z","iopub.execute_input":"2022-05-10T18:09:40.453591Z","iopub.status.idle":"2022-05-10T18:09:42.130389Z","shell.execute_reply.started":"2022-05-10T18:09:40.453557Z","shell.execute_reply":"2022-05-10T18:09:42.129550Z"},"trusted":true},"execution_count":null,"outputs":[]}]}