{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Experiment**\n**Predict next article**\n\nCreate the dataset by adding the next article bought by the customer, creating embeddings for articles by applying PCA to one hot encoded article features.\n\n**Expected reason of low accuracy is the high variance in target variable as, only considering the first 100K transactions, there is around 14K unique values of next_article_id with almost half of them only occur once.**","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom sklearn.tree import DecisionTreeClassifier as dtc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import *\nfrom sklearn.decomposition import PCA","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-20T11:16:12.282998Z","iopub.execute_input":"2022-03-20T11:16:12.283263Z","iopub.status.idle":"2022-03-20T11:16:12.288758Z","shell.execute_reply.started":"2022-03-20T11:16:12.283230Z","shell.execute_reply":"2022-03-20T11:16:12.288138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', chunksize=100000, dtype=str)\narticles = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/articles.csv', dtype=str)\nusers = next(df)\nusers = users.merge(articles, on='article_id')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:28:50.513504Z","iopub.execute_input":"2022-03-20T10:28:50.513867Z","iopub.status.idle":"2022-03-20T10:28:52.218116Z","shell.execute_reply.started":"2022-03-20T10:28:50.513831Z","shell.execute_reply":"2022-03-20T10:28:52.216948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_by_customer = users.groupby('customer_id')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:28:52.953480Z","iopub.execute_input":"2022-03-20T10:28:52.954545Z","iopub.status.idle":"2022-03-20T10:28:52.959747Z","shell.execute_reply.started":"2022-03-20T10:28:52.954492Z","shell.execute_reply":"2022-03-20T10:28:52.958550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Create next_feature for every feature of each article for experimentation**","metadata":{}},{"cell_type":"code","source":"groups = []\nfor key in group_by_customer.groups.keys():\n    group = group_by_customer.get_group(key).sort_values(by='t_dat')\n    for column in group.columns:\n        group['next_{}'.format(column)] = group[column].shift(-1)\n        \n    group = group.drop_duplicates(subset=['customer_id', 'article_id']).iloc[:-1]\n    groups.append(group)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:29:06.224103Z","iopub.execute_input":"2022-03-20T10:29:06.224467Z","iopub.status.idle":"2022-03-20T10:37:07.744899Z","shell.execute_reply.started":"2022-03-20T10:29:06.224429Z","shell.execute_reply":"2022-03-20T10:37:07.743873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_by_customers= pd.concat(groups)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:42:30.363474Z","iopub.execute_input":"2022-03-20T10:42:30.364401Z","iopub.status.idle":"2022-03-20T10:42:35.137129Z","shell.execute_reply.started":"2022-03-20T10:42:30.364341Z","shell.execute_reply":"2022-03-20T10:42:35.136169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_by_customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:55:12.739984Z","iopub.execute_input":"2022-03-20T10:55:12.740955Z","iopub.status.idle":"2022-03-20T10:55:12.767050Z","shell.execute_reply.started":"2022-03-20T10:55:12.740905Z","shell.execute_reply":"2022-03-20T10:55:12.766094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_by_customers.nunique()[['customer_id', 'article_id', 'next_article_id']]","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:56:45.629367Z","iopub.execute_input":"2022-03-20T10:56:45.630785Z","iopub.status.idle":"2022-03-20T10:56:45.993219Z","shell.execute_reply.started":"2022-03-20T10:56:45.630690Z","shell.execute_reply":"2022-03-20T10:56:45.991931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_by_customers.to_csv('df_with_next_article.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:42:47.284926Z","iopub.execute_input":"2022-03-20T10:42:47.285282Z","iopub.status.idle":"2022-03-20T10:42:50.159076Z","shell.execute_reply.started":"2022-03-20T10:42:47.285233Z","shell.execute_reply":"2022-03-20T10:42:50.157945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = grouped_by_customers[['t_dat', 'customer_id', 'article_id', 'prod_name', 'product_type_name',\n       'product_group_name', \n       'graphical_appearance_name', 'colour_group_name',\n       'perceived_colour_value_name',\n       'perceived_colour_master_name',\n       'department_name', 'index_name',\n       'index_group_name', 'section_name',\n       'garment_group_name',\n       'next_article_id']].reset_index().drop('index', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:00:56.140203Z","iopub.execute_input":"2022-03-20T11:00:56.140628Z","iopub.status.idle":"2022-03-20T11:00:56.181522Z","shell.execute_reply.started":"2022-03-20T11:00:56.140584Z","shell.execute_reply":"2022-03-20T11:00:56.180549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:00:56.399917Z","iopub.execute_input":"2022-03-20T11:00:56.400201Z","iopub.status.idle":"2022-03-20T11:00:56.420013Z","shell.execute_reply.started":"2022-03-20T11:00:56.400171Z","shell.execute_reply":"2022-03-20T11:00:56.419060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_features = df[[\n       'product_group_name', \n       'graphical_appearance_name', 'colour_group_name',\n       'perceived_colour_value_name',\n       'perceived_colour_master_name',\n       'department_name', 'index_name',\n       'index_group_name', 'section_name',\n       'garment_group_name']]\ndummies = pd.get_dummies(input_features)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:00:58.193393Z","iopub.execute_input":"2022-03-20T11:00:58.193755Z","iopub.status.idle":"2022-03-20T11:00:58.434677Z","shell.execute_reply.started":"2022-03-20T11:00:58.193705Z","shell.execute_reply":"2022-03-20T11:00:58.433863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_components = 100\npca = PCA(n_components)\npca.fit(dummies)\npca.explained_variance_ratio_.sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:00:58.571528Z","iopub.execute_input":"2022-03-20T11:00:58.571812Z","iopub.status.idle":"2022-03-20T11:01:02.600027Z","shell.execute_reply.started":"2022-03-20T11:00:58.571781Z","shell.execute_reply":"2022-03-20T11:01:02.599207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_var_cumul = np.cumsum(pca.explained_variance_ratio_)\nplt.fill_between(range(1, 101), exp_var_cumul)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:01:02.601765Z","iopub.execute_input":"2022-03-20T11:01:02.602489Z","iopub.status.idle":"2022-03-20T11:01:02.744169Z","shell.execute_reply.started":"2022-03-20T11:01:02.602445Z","shell.execute_reply":"2022-03-20T11:01:02.743015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_pca = pca.transform(dummies)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:01:02.745821Z","iopub.execute_input":"2022-03-20T11:01:02.746139Z","iopub.status.idle":"2022-03-20T11:01:03.062930Z","shell.execute_reply.started":"2022-03-20T11:01:02.746098Z","shell.execute_reply":"2022-03-20T11:01:03.061918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ntarget = df.next_article_id\n\nle = LabelEncoder()\ntarget = le.fit_transform(target)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:01:03.065080Z","iopub.execute_input":"2022-03-20T11:01:03.065612Z","iopub.status.idle":"2022-03-20T11:01:03.170572Z","shell.execute_reply.started":"2022-03-20T11:01:03.065565Z","shell.execute_reply":"2022-03-20T11:01:03.169591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(features_pca, target, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:01:05.645517Z","iopub.execute_input":"2022-03-20T11:01:05.645901Z","iopub.status.idle":"2022-03-20T11:01:05.678138Z","shell.execute_reply.started":"2022-03-20T11:01:05.645862Z","shell.execute_reply":"2022-03-20T11:01:05.677058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree = dtc()\n\ntree.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:01:08.333459Z","iopub.execute_input":"2022-03-20T11:01:08.334215Z","iopub.status.idle":"2022-03-20T11:16:07.274698Z","shell.execute_reply.started":"2022-03-20T11:01:08.334162Z","shell.execute_reply":"2022-03-20T11:16:07.273740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree.score(x_train, y_train), tree.score(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T11:20:06.920907Z","iopub.execute_input":"2022-03-20T11:20:06.921308Z","iopub.status.idle":"2022-03-20T11:20:11.245063Z","shell.execute_reply.started":"2022-03-20T11:20:06.921247Z","shell.execute_reply":"2022-03-20T11:20:11.244325Z"},"trusted":true},"execution_count":null,"outputs":[]}]}