{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview of this notebook\nLike the notebook that this is forked from, we're going to use k-means grouping. This time we're grouping customers, rather than articles.\n\nThe forked notebook for k-means for articles: https://www.kaggle.com/code/beezus666/k-means-and-feature-importance-for-articles\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \npd.options.plotting.backend = \"matplotlib\"\nimport matplotlib.pyplot as plt\nfrom datetime import datetime, timedelta\nimport gc\nimport cudf\nimport cupy as cp\nfrom cuml.cluster import KMeans\nfrom cuml.datasets import make_blobs\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-29T05:20:57.906489Z","iopub.execute_input":"2022-04-29T05:20:57.906853Z","iopub.status.idle":"2022-04-29T05:21:01.968514Z","shell.execute_reply.started":"2022-04-29T05:20:57.906768Z","shell.execute_reply":"2022-04-29T05:21:01.967693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create metrics for each customer\nThe end result we're going for here is one line per customer, the columns added will be a count of how many times they bought something in each category.\n\nSo, we'll need to start with grouping transactions like was done in the articles k-means, to result in the number of times something was bought. \n\nThen will find low-ish cardinatlity features in articles that we can one-hot-encode (the k-means feature built last time included). Then group and count those columns for each customer.\n","metadata":{}},{"cell_type":"code","source":"#some nice ideas on reducing memory: https://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/308635\ntransactions = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', parse_dates=['t_dat'])\ntransactions['customer_id'] = transactions['customer_id'].str[-16:].str.hex_to_int().astype('float64')\ntransactions['article_id'] = transactions.article_id.astype('int32')\ntransactions.t_dat = cudf.to_datetime(transactions.t_dat)\ntransactions = transactions[['t_dat','customer_id','article_id']]\n#transactions.to_parquet('train.pqt',index=False)\nprint( transactions.shape )\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:25:16.396093Z","iopub.execute_input":"2022-04-29T05:25:16.396358Z","iopub.status.idle":"2022-04-29T05:25:19.363453Z","shell.execute_reply.started":"2022-04-29T05:25:16.396330Z","shell.execute_reply":"2022-04-29T05:25:19.362738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#low_cardinality.append('article_id')\narticles = cudf.read_parquet('../input/k-means-and-feature-importance-for-articles/articles.parquet', columns = ['perceived_colour_value_id', 'clusters', 'article_id'])\nprint(articles.shape)\narticles.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:25:19.365115Z","iopub.execute_input":"2022-04-29T05:25:19.365882Z","iopub.status.idle":"2022-04-29T05:25:19.436313Z","shell.execute_reply.started":"2022-04-29T05:25:19.365844Z","shell.execute_reply":"2022-04-29T05:25:19.432659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adding data from articles onto transactions\ntransactions = cudf.merge(transactions, articles, on='article_id', how='left')\nprint(transactions.shape)\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:25:19.438249Z","iopub.execute_input":"2022-04-29T05:25:19.438597Z","iopub.status.idle":"2022-04-29T05:25:19.532089Z","shell.execute_reply.started":"2022-04-29T05:25:19.438556Z","shell.execute_reply":"2022-04-29T05:25:19.531375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# don't want to group + sum on date. \n# Maybe should have used date to split off the last week to do a proper train/test split\ntransactions.drop(columns=['t_dat', 'article_id'], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:25:27.907297Z","iopub.execute_input":"2022-04-29T05:25:27.907554Z","iopub.status.idle":"2022-04-29T05:25:27.912269Z","shell.execute_reply.started":"2022-04-29T05:25:27.907525Z","shell.execute_reply":"2022-04-29T05:25:27.911152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_group = transactions.groupby(['customer_id', 'clusters']).size().to_frame('grouped_count').reset_index()\ntransactions_group.rename(columns={\"clusters\": \"article_clusters\"}, inplace=True)\ntransactions_group.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:26:38.209969Z","iopub.execute_input":"2022-04-29T05:26:38.210232Z","iopub.status.idle":"2022-04-29T05:26:38.290314Z","shell.execute_reply.started":"2022-04-29T05:26:38.210204Z","shell.execute_reply":"2022-04-29T05:26:38.289529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f'group by: {len(transactions_group):,}, original: {len(transactions):,}'","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:26:41.114624Z","iopub.execute_input":"2022-04-29T05:26:41.115188Z","iopub.status.idle":"2022-04-29T05:26:41.120333Z","shell.execute_reply.started":"2022-04-29T05:26:41.115151Z","shell.execute_reply":"2022-04-29T05:26:41.119659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#customers = cudf.read_parquet('../input/radek-customers-parquet-output/customers.parquet')\ncols_to_use = ['customer_id', 'age']\ncustomers = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv', usecols = cols_to_use)\ncustomers.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:26:47.185334Z","iopub.execute_input":"2022-04-29T05:26:47.185604Z","iopub.status.idle":"2022-04-29T05:26:47.344273Z","shell.execute_reply.started":"2022-04-29T05:26:47.185573Z","shell.execute_reply":"2022-04-29T05:26:47.343549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers['customer_id'] = customers['customer_id'].str[-16:].str.hex_to_int().astype('float64')\ncustomers_grouped_trans = cudf.merge(customers, transactions_group, on='customer_id', how='left')\ncustomers_grouped_trans.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:26:48.550048Z","iopub.execute_input":"2022-04-29T05:26:48.550308Z","iopub.status.idle":"2022-04-29T05:26:48.592510Z","shell.execute_reply.started":"2022-04-29T05:26:48.550279Z","shell.execute_reply":"2022-04-29T05:26:48.591795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_grouped_trans.dtypes, customers_grouped_trans.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:27:59.300534Z","iopub.execute_input":"2022-04-29T05:27:59.301216Z","iopub.status.idle":"2022-04-29T05:27:59.307406Z","shell.execute_reply.started":"2022-04-29T05:27:59.301180Z","shell.execute_reply":"2022-04-29T05:27:59.306611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# K-means cluster\nUsing K-means to create a feature on articles","metadata":{}},{"cell_type":"code","source":"# k-means can't take int, needs float\ngrouped_cols = customers_grouped_trans.columns\nfor i in grouped_cols:\n    customers_grouped_trans[i] = customers_grouped_trans[i].astype(float)\n\n    \ncustomers_grouped_trans.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:28:10.729526Z","iopub.execute_input":"2022-04-29T05:28:10.729959Z","iopub.status.idle":"2022-04-29T05:28:10.740922Z","shell.execute_reply.started":"2022-04-29T05:28:10.729922Z","shell.execute_reply":"2022-04-29T05:28:10.740116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_grouped_trans.fillna(0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:28:12.282429Z","iopub.execute_input":"2022-04-29T05:28:12.282946Z","iopub.status.idle":"2022-04-29T05:28:12.298648Z","shell.execute_reply.started":"2022-04-29T05:28:12.282909Z","shell.execute_reply":"2022-04-29T05:28:12.297948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# elbow method to determine the best number of clusters\n# so fast on GPU!\nSum_of_squared_distances = []\nK = range(1, 7)\nfor num_clusters in K :\n kmeans = KMeans(n_clusters=num_clusters)\n kmeans.fit(customers_grouped_trans)\n Sum_of_squared_distances.append(kmeans.inertia_)\nplt.plot(K,Sum_of_squared_distances,'bx-')\nplt.xlabel('Values of K') \nplt.ylabel('Sum of squared distances/Inertia') \nplt.title('Elbow Method For Optimal k')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:28:58.722298Z","iopub.execute_input":"2022-04-29T05:28:58.722554Z","iopub.status.idle":"2022-04-29T05:29:19.761214Z","shell.execute_reply.started":"2022-04-29T05:28:58.722526Z","shell.execute_reply":"2022-04-29T05:29:19.760543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# elbow looks like 3\nkmeans_float = KMeans(n_clusters=3)\nkmeans_fit = kmeans_float.fit(customers_grouped_trans)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:30.343009Z","iopub.execute_input":"2022-04-29T05:29:30.343266Z","iopub.status.idle":"2022-04-29T05:29:32.217886Z","shell.execute_reply.started":"2022-04-29T05:29:30.343237Z","shell.execute_reply":"2022-04-29T05:29:32.217022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"labels:\")\nprint(kmeans_float.labels_)\nprint(\"cluster_centers:\")\nprint(kmeans_float.cluster_centers_)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:32.219352Z","iopub.execute_input":"2022-04-29T05:29:32.219601Z","iopub.status.idle":"2022-04-29T05:29:32.471470Z","shell.execute_reply.started":"2022-04-29T05:29:32.219570Z","shell.execute_reply":"2022-04-29T05:29:32.470323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kmeans_float.fit_predict(customers_grouped_trans)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:34.491358Z","iopub.execute_input":"2022-04-29T05:29:34.491605Z","iopub.status.idle":"2022-04-29T05:29:35.656300Z","shell.execute_reply.started":"2022-04-29T05:29:34.491578Z","shell.execute_reply":"2022-04-29T05:29:35.655512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = kmeans_float.labels_\n\n#Glue back to originaal data\ncustomers_grouped_trans['clusters'] = labels","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:37.355418Z","iopub.execute_input":"2022-04-29T05:29:37.355975Z","iopub.status.idle":"2022-04-29T05:29:37.361014Z","shell.execute_reply.started":"2022-04-29T05:29:37.355932Z","shell.execute_reply":"2022-04-29T05:29:37.360275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_grouped_trans.clusters.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:39.384755Z","iopub.execute_input":"2022-04-29T05:29:39.385347Z","iopub.status.idle":"2022-04-29T05:29:39.414222Z","shell.execute_reply.started":"2022-04-29T05:29:39.385310Z","shell.execute_reply":"2022-04-29T05:29:39.413475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_grouped_trans.tail()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:29:45.472306Z","iopub.execute_input":"2022-04-29T05:29:45.472552Z","iopub.status.idle":"2022-04-29T05:29:45.497788Z","shell.execute_reply.started":"2022-04-29T05:29:45.472524Z","shell.execute_reply":"2022-04-29T05:29:45.496995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_grouped_trans.to_parquet('customers_group.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:30:01.447430Z","iopub.execute_input":"2022-04-29T05:30:01.447705Z","iopub.status.idle":"2022-04-29T05:30:01.600810Z","shell.execute_reply.started":"2022-04-29T05:30:01.447654Z","shell.execute_reply":"2022-04-29T05:30:01.600067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make transactions DF\nFor future notebook, make transactions DF to determine if ","metadata":{}},{"cell_type":"code","source":"transactions = cudf.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', parse_dates=['t_dat'])\ntransactions['customer_id'] = transactions['customer_id'].str[-16:].str.hex_to_int().astype('float64')\ntransactions['article_id'] = transactions.article_id.astype('int32')\ntransactions.t_dat = cudf.to_datetime(transactions.t_dat)\ntransactions = transactions[['t_dat','customer_id','article_id']]\n#transactions.to_parquet('train.pqt',index=False)\nprint( transactions.shape )\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:34:16.937187Z","iopub.execute_input":"2022-04-29T05:34:16.937469Z","iopub.status.idle":"2022-04-29T05:34:19.824618Z","shell.execute_reply.started":"2022-04-29T05:34:16.937437Z","shell.execute_reply":"2022-04-29T05:34:19.823927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = cudf.merge(transactions, customers_grouped_trans, on='customer_id', how='left')\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:34:25.728296Z","iopub.execute_input":"2022-04-29T05:34:25.728832Z","iopub.status.idle":"2022-04-29T05:34:25.947510Z","shell.execute_reply.started":"2022-04-29T05:34:25.728794Z","shell.execute_reply":"2022-04-29T05:34:25.946734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.isnull().sum(axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:34:35.358488Z","iopub.execute_input":"2022-04-29T05:34:35.358757Z","iopub.status.idle":"2022-04-29T05:34:35.460307Z","shell.execute_reply.started":"2022-04-29T05:34:35.358726Z","shell.execute_reply":"2022-04-29T05:34:35.459494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"t_dat                 0\ncustomer_id           0\narticle_id            0\nFN             18209837\nActive         18412468\nage              140258","metadata":{}},{"cell_type":"code","source":"#trans_pd = transactions.to_pandas()\n#trans_pd.to_parquet('transactions_articles_customers.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:40:17.456652Z","iopub.execute_input":"2022-04-29T05:40:17.457478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transactions.to_parquet('transactions_articles_customers.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T05:35:58.520775Z","iopub.execute_input":"2022-04-29T05:35:58.521568Z","iopub.status.idle":"2022-04-29T05:35:59.019368Z","shell.execute_reply.started":"2022-04-29T05:35:58.521519Z","shell.execute_reply":"2022-04-29T05:35:59.018524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}