{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import csv\nimport pandas as pd\nimport missingno as msno\nimport numpy as np\nfrom pandas import Series, DataFrame\nimport sklearn\nfrom sklearn import preprocessing\nfrom tqdm import tqdm\nimport gc\n%precision 3","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:08:20.337393Z","iopub.execute_input":"2022-03-12T01:08:20.338118Z","iopub.status.idle":"2022-03-12T01:08:21.502692Z","shell.execute_reply.started":"2022-03-12T01:08:20.337999Z","shell.execute_reply":"2022-03-12T01:08:21.501820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\narticles = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\ntransactions_train = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:08:21.505229Z","iopub.execute_input":"2022-03-12T01:08:21.505840Z","iopub.status.idle":"2022-03-12T01:09:32.822263Z","shell.execute_reply.started":"2022-03-12T01:08:21.505791Z","shell.execute_reply":"2022-03-12T01:09:32.821310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\nprint(\"articles\", articles.shape)\nprint(\"customers\", customers.shape)\nprint(\"transactions_train\", transactions_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:32.823524Z","iopub.execute_input":"2022-03-12T01:09:32.823738Z","iopub.status.idle":"2022-03-12T01:09:32.830144Z","shell.execute_reply.started":"2022-03-12T01:09:32.823713Z","shell.execute_reply":"2022-03-12T01:09:32.829221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check missing\nmsno.matrix(customers)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-12T01:09:32.831392Z","iopub.execute_input":"2022-03-12T01:09:32.831672Z","iopub.status.idle":"2022-03-12T01:09:37.867868Z","shell.execute_reply.started":"2022-03-12T01:09:32.831641Z","shell.execute_reply":"2022-03-12T01:09:37.867076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop columns to avoid Multicollinearity\n# It seems that \"product_type_no\", \"product_type_name\" are chosen 1\n# I chose \"product_type_no\" because of handle easily\n# Drop \"product_type_name\"\narticles_1 = articles[[\"product_type_no\", \"product_type_name\"]]\narticles_1.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.869912Z","iopub.execute_input":"2022-03-12T01:09:37.870308Z","iopub.status.idle":"2022-03-12T01:09:37.895645Z","shell.execute_reply.started":"2022-03-12T01:09:37.870273Z","shell.execute_reply":"2022-03-12T01:09:37.894723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same as the above theory, see \"index_code\", \"index_name\"\n# I chose \"index_code\" because of handle easily\n# Drop \"index_name\"\narticles_2 = articles[[\"index_code\", \"index_name\"]]\narticles_2.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.896907Z","iopub.execute_input":"2022-03-12T01:09:37.897664Z","iopub.status.idle":"2022-03-12T01:09:37.909851Z","shell.execute_reply.started":"2022-03-12T01:09:37.897619Z","shell.execute_reply":"2022-03-12T01:09:37.909207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"section_name\" is more detail than \"index_group_name\"\n# Drop \"index_group_name\"\narticles_3 = articles[[\"index_group_name\", \"section_name\"]]\narticles_3.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.910946Z","iopub.execute_input":"2022-03-12T01:09:37.911453Z","iopub.status.idle":"2022-03-12T01:09:37.929695Z","shell.execute_reply.started":"2022-03-12T01:09:37.911408Z","shell.execute_reply":"2022-03-12T01:09:37.928700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same as the above theory, see \"garment_group_name\", \"detail_desc\", \"prod_name\"\n# Drop \"detail_desc\" and \"prod_name\"\narticles_4 = articles[[\"garment_group_name\", \"detail_desc\",\"prod_name\"]]\narticles_4.head(5)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-12T01:09:37.930954Z","iopub.execute_input":"2022-03-12T01:09:37.931170Z","iopub.status.idle":"2022-03-12T01:09:37.948011Z","shell.execute_reply.started":"2022-03-12T01:09:37.931146Z","shell.execute_reply":"2022-03-12T01:09:37.947096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same as the above theory, see \"color_group_code\", \"color_group_name\", \"prod_name\"\n# \"perceived_colour_master_name\", \"perceived_colour_value_name\"\n# Drop \"colour_group_name\", \"preceived_colour_master_name\", \"perceived_colour_value_name\"\n# \"department_name\" and  \"garment_group_name\"\narticles_5 = articles[[\"colour_group_code\", \"colour_group_name\", \"perceived_colour_master_name\",\n                       \"perceived_colour_value_name\", \"department_name\", \"garment_group_name\"]]\narticles_5.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.949176Z","iopub.execute_input":"2022-03-12T01:09:37.949423Z","iopub.status.idle":"2022-03-12T01:09:37.966331Z","shell.execute_reply.started":"2022-03-12T01:09:37.949393Z","shell.execute_reply":"2022-03-12T01:09:37.965538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make new articles data\n# drop above data \"product_type_name\", \"index_name\", \"index_group_name\", \"detail_desc\", \"prod_name\"\n# \"colour_group_name\", \"preceived_colour_master_name\", \"perceived_colour_value_name\"\n# \"department_name\" and \"garment_group_name\"\narticles_new = articles.drop([\"product_type_name\", \"index_name\", \"index_group_name\",\n                              \"detail_desc\", \"prod_name\", \"colour_group_name\",\n                              \"perceived_colour_master_name\", \"perceived_colour_value_name\",\n                             \"department_name\", \"garment_group_name\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.968008Z","iopub.execute_input":"2022-03-12T01:09:37.968618Z","iopub.status.idle":"2022-03-12T01:09:37.982253Z","shell.execute_reply.started":"2022-03-12T01:09:37.968575Z","shell.execute_reply":"2022-03-12T01:09:37.981336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Customer status like is no mean to predict what product will be bought\n# we don't know what the \"fashion_news\" is written\n# Drop \"FN\", \"Active\", \"clubmember_status\", \"fashion_news_frequency\" and \"postal_code\"\ncustomers_new = customers.drop([\"FN\", \"Active\", \"club_member_status\",\n                             \"fashion_news_frequency\", \"postal_code\"],axis=1)\n\n# Few \"age\" data is missing\n# So, filling any age is less impact\ncustomers_new[\"age\"] = customers_new[\"age\"].fillna(customers_new[\"age\"].mean())","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:37.983438Z","iopub.execute_input":"2022-03-12T01:09:37.983786Z","iopub.status.idle":"2022-03-12T01:09:38.027371Z","shell.execute_reply.started":"2022-03-12T01:09:37.983757Z","shell.execute_reply":"2022-03-12T01:09:38.026549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Binary data \"sales_channel_id\" is not important\n# Because we predict variety of products that customer will buy\ntransactions_train_new = transactions_train.drop([\"sales_channel_id\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:38.028620Z","iopub.execute_input":"2022-03-12T01:09:38.029635Z","iopub.status.idle":"2022-03-12T01:09:38.988120Z","shell.execute_reply.started":"2022-03-12T01:09:38.029597Z","shell.execute_reply":"2022-03-12T01:09:38.987209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare data shape\nprint(\"------articles------\")\nprint(\"Before\", articles.shape)\nprint(\"After\", articles_new.shape)\nprint(\"------customers------\")\nprint(\"Before\", customers.shape)\nprint(\"After\", customers_new.shape)\nprint(\"------transactions_train------\")\nprint(\"Before\", transactions_train.shape)\nprint(\"After\", transactions_train_new.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:38.989289Z","iopub.execute_input":"2022-03-12T01:09:38.989819Z","iopub.status.idle":"2022-03-12T01:09:38.997486Z","shell.execute_reply.started":"2022-03-12T01:09:38.989784Z","shell.execute_reply":"2022-03-12T01:09:38.996531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del articles, customers, transactions_train, articles_1, articles_2, articles_3, articles_4, articles_5\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:39.001832Z","iopub.execute_input":"2022-03-12T01:09:39.002254Z","iopub.status.idle":"2022-03-12T01:09:39.449544Z","shell.execute_reply.started":"2022-03-12T01:09:39.002199Z","shell.execute_reply":"2022-03-12T01:09:39.448755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# letter and long data is difficult to handle\n# LabelEncoder\nle = preprocessing.LabelEncoder()\n\narticles_new[\"product_group_name\"] = le.fit_transform(articles_new[\"product_group_name\"])\narticles_new[\"graphical_appearance_name\"] = le.fit_transform(articles_new[\"graphical_appearance_name\"])\narticles_new[\"section_name\"] = le.fit_transform(articles_new[\"section_name\"])\n\narticles_new[\"product_group_name\"] = articles_new[\"product_group_name\"].astype(\"int32\")\narticles_new[\"graphical_appearance_name\"] = articles_new[\"graphical_appearance_name\"].astype(\"int32\")\narticles_new[\"section_name\"] = articles_new[\"section_name\"].astype(\"int32\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:39.450824Z","iopub.execute_input":"2022-03-12T01:09:39.451033Z","iopub.status.idle":"2022-03-12T01:09:39.562022Z","shell.execute_reply.started":"2022-03-12T01:09:39.451007Z","shell.execute_reply":"2022-03-12T01:09:39.561269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge above data\nmerge_data = pd.merge(articles_new, transactions_train_new, on=\"article_id\", copy=False)\ndel articles_new, transactions_train_new\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:09:39.563127Z","iopub.execute_input":"2022-03-12T01:09:39.563465Z","iopub.status.idle":"2022-03-12T01:10:04.753939Z","shell.execute_reply.started":"2022-03-12T01:09:39.563436Z","shell.execute_reply":"2022-03-12T01:10:04.753040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# see customer bought in around September\nmerge_data[\"month\"] = pd.to_datetime(merge_data[\"t_dat\"]).dt.strftime(\"%m\")","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:10:04.755272Z","iopub.execute_input":"2022-03-12T01:10:04.755590Z","iopub.status.idle":"2022-03-12T01:14:03.238651Z","shell.execute_reply.started":"2022-03-12T01:10:04.755546Z","shell.execute_reply":"2022-03-12T01:14:03.237716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_data = merge_data.drop([\"t_dat\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:14:03.239896Z","iopub.execute_input":"2022-03-12T01:14:03.240122Z","iopub.status.idle":"2022-03-12T01:14:18.749592Z","shell.execute_reply.started":"2022-03-12T01:14:03.240095Z","shell.execute_reply":"2022-03-12T01:14:18.748842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_month_list = [\"01\", \"02\", \"03\", \"04\", \"05\", \"06\", \"07\", \"11\", \"12\"]\nfor mon in tqdm(drop_month_list):\n    drop_index_month = merge_data.index[merge_data[\"month\"] == mon]\n    merge_data = merge_data.drop(drop_index_month)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:14:18.751314Z","iopub.execute_input":"2022-03-12T01:14:18.751671Z","iopub.status.idle":"2022-03-12T01:15:56.584141Z","shell.execute_reply.started":"2022-03-12T01:14:18.751629Z","shell.execute_reply":"2022-03-12T01:15:56.583038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_data = merge_data.drop([\"month\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:15:56.585829Z","iopub.execute_input":"2022-03-12T01:15:56.586150Z","iopub.status.idle":"2022-03-12T01:15:57.723027Z","shell.execute_reply.started":"2022-03-12T01:15:56.586107Z","shell.execute_reply":"2022-03-12T01:15:57.721861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_data = pd.merge(customers_new, merge_data, on=\"customer_id\", copy=False)\n\ndel customers_new\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:15:57.724274Z","iopub.execute_input":"2022-03-12T01:15:57.724634Z","iopub.status.idle":"2022-03-12T01:16:13.538153Z","shell.execute_reply.started":"2022-03-12T01:15:57.724599Z","shell.execute_reply":"2022-03-12T01:16:13.537295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:16:13.539394Z","iopub.execute_input":"2022-03-12T01:16:13.539670Z","iopub.status.idle":"2022-03-12T01:16:13.561144Z","shell.execute_reply.started":"2022-03-12T01:16:13.539640Z","shell.execute_reply":"2022-03-12T01:16:13.560418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:16:13.562602Z","iopub.execute_input":"2022-03-12T01:16:13.563203Z","iopub.status.idle":"2022-03-12T01:16:13.568961Z","shell.execute_reply.started":"2022-03-12T01:16:13.563165Z","shell.execute_reply":"2022-03-12T01:16:13.568050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I want to drop data more.\nBelow may be not appropriate processing.\nAs we see Discussion, few people bought same products and products are large variation.\nwe may drop data that same customer bought twice and unpopular products.","metadata":{}},{"cell_type":"code","source":"# drop buying twice data\nmerge_data = merge_data.drop_duplicates(subset=[\"customer_id\", \"article_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:16:13.570620Z","iopub.execute_input":"2022-03-12T01:16:13.571480Z","iopub.status.idle":"2022-03-12T01:16:17.780666Z","shell.execute_reply.started":"2022-03-12T01:16:13.571441Z","shell.execute_reply":"2022-03-12T01:16:17.779997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop unpopluar products\nfeature, count = np.unique(merge_data[\"article_id\"], return_counts=True)\nunpopular = feature[count <=1 ]","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:16:17.782038Z","iopub.execute_input":"2022-03-12T01:16:17.782467Z","iopub.status.idle":"2022-03-12T01:16:18.296618Z","shell.execute_reply.started":"2022-03-12T01:16:17.782436Z","shell.execute_reply":"2022-03-12T01:16:18.295949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_index_list = []\nfor i in tqdm(range(unpopular.size)):\n    drop_index = merge_data.index[merge_data[\"article_id\"] == unpopular[i]]\n    drop_index_list.append(drop_index)\n\nfor i in tqdm(range(len(drop_index_list))):\n    merge_data = merge_data.drop(drop_index_list[i], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T01:16:18.298015Z","iopub.execute_input":"2022-03-12T01:16:18.298447Z","iopub.status.idle":"2022-03-12T04:56:21.815352Z","shell.execute_reply.started":"2022-03-12T01:16:18.298415Z","shell.execute_reply":"2022-03-12T04:56:21.812726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Data shape after this process\", merge_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-12T04:56:21.822876Z","iopub.execute_input":"2022-03-12T04:56:21.823373Z","iopub.status.idle":"2022-03-12T04:56:21.836972Z","shell.execute_reply.started":"2022-03-12T04:56:21.823342Z","shell.execute_reply":"2022-03-12T04:56:21.836046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}