{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-26T12:03:15.803304Z","iopub.execute_input":"2022-03-26T12:03:15.804566Z","iopub.status.idle":"2022-03-26T12:03:17.008293Z","shell.execute_reply.started":"2022-03-26T12:03:15.804424Z","shell.execute_reply":"2022-03-26T12:03:17.007359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data into dataframes\npath = \"../input/h-and-m-personalized-fashion-recommendations/\"\n\narticles = pd.read_csv(path + \"articles.csv\")\ncustomers = pd.read_csv(path + \"customers.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:17.009903Z","iopub.execute_input":"2022-03-26T12:03:17.011741Z","iopub.status.idle":"2022-03-26T12:03:24.995685Z","shell.execute_reply.started":"2022-03-26T12:03:17.011698Z","shell.execute_reply":"2022-03-26T12:03:24.994727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Let's see the first five rows from each dataframe to make an idea about our data.","metadata":{}},{"cell_type":"code","source":"articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:24.997163Z","iopub.execute_input":"2022-03-26T12:03:24.997413Z","iopub.status.idle":"2022-03-26T12:03:25.006889Z","shell.execute_reply.started":"2022-03-26T12:03:24.997376Z","shell.execute_reply":"2022-03-26T12:03:25.005841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.008631Z","iopub.execute_input":"2022-03-26T12:03:25.009348Z","iopub.status.idle":"2022-03-26T12:03:25.030692Z","shell.execute_reply.started":"2022-03-26T12:03:25.009319Z","shell.execute_reply":"2022-03-26T12:03:25.029379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.032709Z","iopub.execute_input":"2022-03-26T12:03:25.03308Z","iopub.status.idle":"2022-03-26T12:03:25.082385Z","shell.execute_reply.started":"2022-03-26T12:03:25.033049Z","shell.execute_reply":"2022-03-26T12:03:25.081486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.083516Z","iopub.execute_input":"2022-03-26T12:03:25.083729Z","iopub.status.idle":"2022-03-26T12:03:25.096281Z","shell.execute_reply.started":"2022-03-26T12:03:25.083704Z","shell.execute_reply":"2022-03-26T12:03:25.095712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Looks like we some missing values\n#### Let's see ......","metadata":{}},{"cell_type":"code","source":"customers.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.097318Z","iopub.execute_input":"2022-03-26T12:03:25.097772Z","iopub.status.idle":"2022-03-26T12:03:25.385513Z","shell.execute_reply.started":"2022-03-26T12:03:25.09774Z","shell.execute_reply":"2022-03-26T12:03:25.384186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We have missing values for all columns except 'customer_id' and 'postal_code'.\n* Many customers are missing 'FN' and 'Active'. Maybe there are just ones so we can put zeros instead of NaN's and treat them as boolean columns (i.e. if Active is 1 this means yes the customer is active and if Active is 0 that means the customer is not active; same for FN)","metadata":{}},{"cell_type":"code","source":"print(customers.FN.mean(skipna=True))\nprint(customers.Active.mean(skipna=True)) ","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.386644Z","iopub.execute_input":"2022-03-26T12:03:25.386885Z","iopub.status.idle":"2022-03-26T12:03:25.40704Z","shell.execute_reply.started":"2022-03-26T12:03:25.38685Z","shell.execute_reply":"2022-03-26T12:03:25.405837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The non-NA mean is 1.0 so we can deduce that there are only ones. Let's fill with zeros where we have NaN.","metadata":{}},{"cell_type":"code","source":"customers['FN'] = customers['FN'].fillna(0)\ncustomers['Active'] = customers['Active'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.408073Z","iopub.execute_input":"2022-03-26T12:03:25.408285Z","iopub.status.idle":"2022-03-26T12:03:25.438616Z","shell.execute_reply.started":"2022-03-26T12:03:25.408259Z","shell.execute_reply":"2022-03-26T12:03:25.437347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.info() # yey, no missing values for FN and Active","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.44144Z","iopub.execute_input":"2022-03-26T12:03:25.441706Z","iopub.status.idle":"2022-03-26T12:03:25.694903Z","shell.execute_reply.started":"2022-03-26T12:03:25.441683Z","shell.execute_reply":"2022-03-26T12:03:25.693534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Let's make some simple plots for this two columns","metadata":{}},{"cell_type":"code","source":"x = customers.FN.values\n\nplt.figure(figsize=(8, 6), dpi=80)\nplt.hist(x)\nplt.xlabel('FN')\nplt.ylabel('Number of people')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.69618Z","iopub.execute_input":"2022-03-26T12:03:25.696423Z","iopub.status.idle":"2022-03-26T12:03:25.941187Z","shell.execute_reply.started":"2022-03-26T12:03:25.696393Z","shell.execute_reply":"2022-03-26T12:03:25.93949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = customers.Active.values\n\nplt.figure(figsize=(8, 6), dpi=80)\nplt.hist(x)\nplt.xlabel('Active')\nplt.ylabel('Number of people')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:25.943291Z","iopub.execute_input":"2022-03-26T12:03:25.943641Z","iopub.status.idle":"2022-03-26T12:03:26.174327Z","shell.execute_reply.started":"2022-03-26T12:03:25.943605Z","shell.execute_reply":"2022-03-26T12:03:26.17355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The two histograms seems to be the same. Maybe there is no difference between FN and Actve\n","metadata":{}},{"cell_type":"code","source":"s = customers.FN + customers.Active\nprint(customers.FN.unique())\nprint(customers.Active.unique())\nprint(s.unique())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.17559Z","iopub.execute_input":"2022-03-26T12:03:26.176385Z","iopub.status.idle":"2022-03-26T12:03:26.230398Z","shell.execute_reply.started":"2022-03-26T12:03:26.176342Z","shell.execute_reply":"2022-03-26T12:03:26.22923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* No, I was wrong. There aren't the same.","metadata":{}},{"cell_type":"code","source":"customers.fashion_news_frequency.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.231827Z","iopub.execute_input":"2022-03-26T12:03:26.232029Z","iopub.status.idle":"2022-03-26T12:03:26.306381Z","shell.execute_reply.started":"2022-03-26T12:03:26.232006Z","shell.execute_reply":"2022-03-26T12:03:26.305453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.loc[customers['fashion_news_frequency'] == 'NONE', 'fashion_news_frequency'] = 'None'\ncustomers.fashion_news_frequency = customers.fashion_news_frequency.fillna('None')","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.307508Z","iopub.execute_input":"2022-03-26T12:03:26.307729Z","iopub.status.idle":"2022-03-26T12:03:26.573632Z","shell.execute_reply.started":"2022-03-26T12:03:26.3077Z","shell.execute_reply":"2022-03-26T12:03:26.572491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.fashion_news_frequency.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.575818Z","iopub.execute_input":"2022-03-26T12:03:26.576159Z","iopub.status.idle":"2022-03-26T12:03:26.67137Z","shell.execute_reply.started":"2022-03-26T12:03:26.576121Z","shell.execute_reply":"2022-03-26T12:03:26.670741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.672463Z","iopub.execute_input":"2022-03-26T12:03:26.673198Z","iopub.status.idle":"2022-03-26T12:03:26.748286Z","shell.execute_reply.started":"2022-03-26T12:03:26.673161Z","shell.execute_reply":"2022-03-26T12:03:26.74758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status = customers.club_member_status.fillna(\"NO INFO\")","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.75022Z","iopub.execute_input":"2022-03-26T12:03:26.750516Z","iopub.status.idle":"2022-03-26T12:03:26.850389Z","shell.execute_reply.started":"2022-03-26T12:03:26.75048Z","shell.execute_reply":"2022-03-26T12:03:26.848522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:26.852Z","iopub.execute_input":"2022-03-26T12:03:26.85259Z","iopub.status.idle":"2022-03-26T12:03:27.023047Z","shell.execute_reply.started":"2022-03-26T12:03:26.852524Z","shell.execute_reply":"2022-03-26T12:03:27.022325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.age.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.024261Z","iopub.execute_input":"2022-03-26T12:03:27.024893Z","iopub.status.idle":"2022-03-26T12:03:27.046928Z","shell.execute_reply.started":"2022-03-26T12:03:27.024854Z","shell.execute_reply":"2022-03-26T12:03:27.046035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.age = customers.age.fillna(int(customers.age.mean(skipna=True)))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.048226Z","iopub.execute_input":"2022-03-26T12:03:27.048513Z","iopub.status.idle":"2022-03-26T12:03:27.066215Z","shell.execute_reply.started":"2022-03-26T12:03:27.048473Z","shell.execute_reply":"2022-03-26T12:03:27.065089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.age.unique()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.067742Z","iopub.execute_input":"2022-03-26T12:03:27.068355Z","iopub.status.idle":"2022-03-26T12:03:27.093251Z","shell.execute_reply.started":"2022-03-26T12:03:27.068316Z","shell.execute_reply":"2022-03-26T12:03:27.092046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.095026Z","iopub.execute_input":"2022-03-26T12:03:27.095267Z","iopub.status.idle":"2022-03-26T12:03:27.350531Z","shell.execute_reply.started":"2022-03-26T12:03:27.095242Z","shell.execute_reply":"2022-03-26T12:03:27.349437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We handled all missing values for customers table","metadata":{}},{"cell_type":"code","source":"articles.info() # pretty good for articles","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.352333Z","iopub.execute_input":"2022-03-26T12:03:27.352743Z","iopub.status.idle":"2022-03-26T12:03:27.424777Z","shell.execute_reply.started":"2022-03-26T12:03:27.352705Z","shell.execute_reply":"2022-03-26T12:03:27.423914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.detail_desc = articles.detail_desc.fillna('')","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.425805Z","iopub.execute_input":"2022-03-26T12:03:27.426049Z","iopub.status.idle":"2022-03-26T12:03:27.441607Z","shell.execute_reply.started":"2022-03-26T12:03:27.426014Z","shell.execute_reply":"2022-03-26T12:03:27.440944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.443764Z","iopub.execute_input":"2022-03-26T12:03:27.44405Z","iopub.status.idle":"2022-03-26T12:03:27.625442Z","shell.execute_reply.started":"2022-03-26T12:03:27.444021Z","shell.execute_reply":"2022-03-26T12:03:27.624224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.626758Z","iopub.execute_input":"2022-03-26T12:03:27.627195Z","iopub.status.idle":"2022-03-26T12:03:27.693946Z","shell.execute_reply.started":"2022-03-26T12:03:27.627162Z","shell.execute_reply":"2022-03-26T12:03:27.693422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.698955Z","iopub.execute_input":"2022-03-26T12:03:27.699334Z","iopub.status.idle":"2022-03-26T12:03:27.947282Z","shell.execute_reply.started":"2022-03-26T12:03:27.699299Z","shell.execute_reply":"2022-03-26T12:03:27.946596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### End with missing values","metadata":{}},{"cell_type":"markdown","source":"### Questions","metadata":{}},{"cell_type":"markdown","source":"#### What is the mean age of all customers? Max and Min?","metadata":{}},{"cell_type":"code","source":"customers.age.mean()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.948337Z","iopub.execute_input":"2022-03-26T12:03:27.948589Z","iopub.status.idle":"2022-03-26T12:03:27.956589Z","shell.execute_reply.started":"2022-03-26T12:03:27.948556Z","shell.execute_reply":"2022-03-26T12:03:27.955579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There are people around 36 years old that are registered as customers at H&M\n","metadata":{}},{"cell_type":"code","source":"customers.age.max()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.958242Z","iopub.execute_input":"2022-03-26T12:03:27.958531Z","iopub.status.idle":"2022-03-26T12:03:27.977406Z","shell.execute_reply.started":"2022-03-26T12:03:27.958498Z","shell.execute_reply":"2022-03-26T12:03:27.976016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* This is pretty old, but how many of them are this old?","metadata":{}},{"cell_type":"code","source":"very_old_customers = customers[customers.age == customers.age.max()]\nvery_old_customers.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:27.981462Z","iopub.execute_input":"2022-03-26T12:03:27.981728Z","iopub.status.idle":"2022-03-26T12:03:27.999808Z","shell.execute_reply.started":"2022-03-26T12:03:27.981699Z","shell.execute_reply":"2022-03-26T12:03:27.99878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.age.min()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:28.001696Z","iopub.execute_input":"2022-03-26T12:03:28.002593Z","iopub.status.idle":"2022-03-26T12:03:28.013271Z","shell.execute_reply.started":"2022-03-26T12:03:28.00254Z","shell.execute_reply":"2022-03-26T12:03:28.012193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 6), dpi=80)\nplt.hist(customers.age, bins=np.linspace(customers.age.min(), customers.age.max(), num=100))\nplt.title(\"Customer's age\")\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:17:51.045677Z","iopub.execute_input":"2022-03-26T12:17:51.046042Z","iopub.status.idle":"2022-03-26T12:17:51.5311Z","shell.execute_reply.started":"2022-03-26T12:17:51.046009Z","shell.execute_reply":"2022-03-26T12:17:51.530036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"very_young_customers = customers[customers.age == customers.age.min()]\nvery_young_customers.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:28.014408Z","iopub.execute_input":"2022-03-26T12:03:28.014908Z","iopub.status.idle":"2022-03-26T12:03:28.032437Z","shell.execute_reply.started":"2022-03-26T12:03:28.01488Z","shell.execute_reply":"2022-03-26T12:03:28.031491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Okay, let's divide them into categories like under 20, between 20 and 30, ..., over 60 to answear some questions.","metadata":{}},{"cell_type":"code","source":"age_categories = {}\n\n# tuple (a,b) means interval [a,b)\nage_categories[(0,20)] = customers[customers.age < 20].shape[0] \nage_categories[(20,30)] = customers[customers.age.between(20, 30, inclusive='left')].shape[0]\nage_categories[(30,40)] = customers[customers.age.between(30, 40, inclusive='left')].shape[0]\nage_categories[(40,50)] = customers[customers.age.between(40, 50, inclusive='left')].shape[0]\nage_categories[(50,60)] = customers[customers.age.between(50, 60, inclusive='left')].shape[0]\nage_categories[(60,150)] = customers[customers.age >= 60].shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:28.033587Z","iopub.execute_input":"2022-03-26T12:03:28.033895Z","iopub.status.idle":"2022-03-26T12:03:28.302602Z","shell.execute_reply.started":"2022-03-26T12:03:28.03387Z","shell.execute_reply":"2022-03-26T12:03:28.301236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = list(map(lambda x: \"[{0}, {1})\".format(x[0], x[1]), age_categories.keys()))\ny = age_categories.values()\n\nplt.figure(figsize=(8, 6), dpi=80)\nplt.bar(x, y)\nplt.xlabel('Age')\nplt.ylabel('Number of people')\nplt.title('Customers by age')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:19:21.148748Z","iopub.execute_input":"2022-03-26T12:19:21.150123Z","iopub.status.idle":"2022-03-26T12:19:21.329172Z","shell.execute_reply.started":"2022-03-26T12:19:21.150057Z","shell.execute_reply":"2022-03-26T12:19:21.328261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### So the most people are between 20 and 30, and the mean is not far from 30. That means H&M has a lot of young customers","metadata":{}},{"cell_type":"markdown","source":"#### Now, based on this categories, let's find out the answers at the following questions:\n            1. What category spends a lot of money on clothes?\n            2. In whitch category are the most active members?\n            3. What types of articles are they buying?","metadata":{}},{"cell_type":"markdown","source":"#### 1. What category spends a lot of money on clothes??","metadata":{}},{"cell_type":"code","source":"chunks = pd.read_csv(path + \"transactions_train.csv\", chunksize=5298054) #this will result in 6 chunks","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:28.473161Z","iopub.execute_input":"2022-03-26T12:03:28.473384Z","iopub.status.idle":"2022-03-26T12:03:28.490129Z","shell.execute_reply.started":"2022-03-26T12:03:28.473358Z","shell.execute_reply":"2022-03-26T12:03:28.488756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_list = []\n\nfor chunk in chunks:\n    chunk_filter = chunk.drop(['sales_channel_id', 't_dat'], axis=1) #for the purpose of the analysis \n    \n    chunk_list.append(chunk_filter)\ntransactions = pd.concat(chunk_list)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:03:28.491511Z","iopub.execute_input":"2022-03-26T12:03:28.49175Z","iopub.status.idle":"2022-03-26T12:04:51.978756Z","shell.execute_reply.started":"2022-03-26T12:03:28.491724Z","shell.execute_reply":"2022-03-26T12:04:51.977731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:04:51.980025Z","iopub.execute_input":"2022-03-26T12:04:51.980326Z","iopub.status.idle":"2022-03-26T12:04:51.991596Z","shell.execute_reply.started":"2022-03-26T12:04:51.980288Z","shell.execute_reply":"2022-03-26T12:04:51.990718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_money = transactions.drop('article_id', axis=1)\ntransactions_money_grouped = transactions_money.groupby('customer_id').sum() ","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:04:51.992555Z","iopub.execute_input":"2022-03-26T12:04:51.992785Z","iopub.status.idle":"2022-03-26T12:05:05.031358Z","shell.execute_reply.started":"2022-03-26T12:04:51.992753Z","shell.execute_reply":"2022-03-26T12:05:05.029806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_money_grouped = transactions_money_grouped.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.032895Z","iopub.execute_input":"2022-03-26T12:05:05.033157Z","iopub.status.idle":"2022-03-26T12:05:05.088536Z","shell.execute_reply.started":"2022-03-26T12:05:05.033133Z","shell.execute_reply":"2022-03-26T12:05:05.087118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_money_grouped","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.089479Z","iopub.execute_input":"2022-03-26T12:05:05.089707Z","iopub.status.idle":"2022-03-26T12:05:05.103346Z","shell.execute_reply.started":"2022-03-26T12:05:05.089634Z","shell.execute_reply":"2022-03-26T12:05:05.101939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import floor\n\ndef get_customer_age_category(c_id):\n    left = int(customers.age[customers.customer_id == c_id]) // 10 * 10\n    if left == 10:\n        left = 0\n    if left > 60:\n        left = 60\n    \n    right = left + 10\n    \n    if left == 0:\n        right = 20\n    if left == 60:\n        right = 150\n        \n    # tuple (a,b) means interval [a,b)\n    return (left, right)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.10495Z","iopub.execute_input":"2022-03-26T12:05:05.105253Z","iopub.status.idle":"2022-03-26T12:05:05.120025Z","shell.execute_reply.started":"2022-03-26T12:05:05.105213Z","shell.execute_reply":"2022-03-26T12:05:05.119023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"money_spend = {}\n\n# to much computations, need a better method\n#for index, row in transactions_money_grouped.iterrows():\n    #money_spend[get_customer_age_category(row.customer_id)] = row.price","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.121589Z","iopub.execute_input":"2022-03-26T12:05:05.122135Z","iopub.status.idle":"2022-03-26T12:05:05.140167Z","shell.execute_reply.started":"2022-03-26T12:05:05.122092Z","shell.execute_reply":"2022-03-26T12:05:05.138741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2. In whitch category are the most active members?","metadata":{}},{"cell_type":"code","source":"active_categorires = {}\n\n# tuple (a,b) means interval [a,b)\nactive_categorires[(0,20)] = customers[(customers.age < 20) & (customers.Active)].shape[0] \nactive_categorires[(20,30)] = customers[(customers.age.between(20, 30, inclusive='left')) & (customers.Active)].shape[0]\nactive_categorires[(30,40)] = customers[(customers.age.between(30, 40, inclusive='left')) & (customers.Active == 1.0)].shape[0]\nactive_categorires[(40,50)] = customers[(customers.age.between(40, 50, inclusive='left')) & (customers.Active == 1.0)].shape[0]\nactive_categorires[(50,60)] = customers[(customers.age.between(50, 60, inclusive='left')) & (customers.Active == 1.0)].shape[0]\nactive_categorires[(60,150)] = customers[(customers.age >= 60) & (customers.Active == 1.0)].shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.141892Z","iopub.execute_input":"2022-03-26T12:05:05.14218Z","iopub.status.idle":"2022-03-26T12:05:05.368642Z","shell.execute_reply.started":"2022-03-26T12:05:05.142142Z","shell.execute_reply":"2022-03-26T12:05:05.367217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = list(map(lambda x: \"[{0}, {1})\".format(x[0], x[1]), active_categorires.keys()))\ny = active_categorires.values()\n\nplt.figure(figsize=(8, 6), dpi=80)\nplt.bar(x, y)\nplt.xlabel('Age category')\nplt.ylabel('Number of active people')\nplt.title('Active customers by age category')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.370518Z","iopub.execute_input":"2022-03-26T12:05:05.370779Z","iopub.status.idle":"2022-03-26T12:05:05.565737Z","shell.execute_reply.started":"2022-03-26T12:05:05.370754Z","shell.execute_reply":"2022-03-26T12:05:05.564349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### As we would expect, the majority of people who are active customers has the age between 20, inlcusive, and 30.","metadata":{}},{"cell_type":"markdown","source":"#### 3. What types of articles are they buying?","metadata":{}},{"cell_type":"code","source":"transations_articles = transactions.drop('price', axis=1)\ntransations_articles_grouped = transations_articles.groupby('customer_id')['article_id'].agg(list)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:05.56697Z","iopub.execute_input":"2022-03-26T12:05:05.567171Z","iopub.status.idle":"2022-03-26T12:05:43.499294Z","shell.execute_reply.started":"2022-03-26T12:05:05.567146Z","shell.execute_reply":"2022-03-26T12:05:43.497066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transations_articles_grouped = transations_articles_grouped.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.501092Z","iopub.execute_input":"2022-03-26T12:05:43.501309Z","iopub.status.idle":"2022-03-26T12:05:43.67023Z","shell.execute_reply.started":"2022-03-26T12:05:43.501283Z","shell.execute_reply":"2022-03-26T12:05:43.668633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_category = {}\n\n# very intensive task\n'''\nfor index, row in transations_articles_grouped.iterrows():\n    articles_category[get_customer_age_category(row['customer_id'])] = articles.prod_name[\n        articles.apply(lambda r: r['article_id'] in row['article_id'], axis=1)\n    ]\n'''","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.672368Z","iopub.execute_input":"2022-03-26T12:05:43.672664Z","iopub.status.idle":"2022-03-26T12:05:43.679743Z","shell.execute_reply.started":"2022-03-26T12:05:43.67262Z","shell.execute_reply":"2022-03-26T12:05:43.678826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Let's see if Active == 1 is the same as club_member_status == 'Active'. If this is true we can drop Active column from customers DataFrame","metadata":{}},{"cell_type":"code","source":" active_members = customers.Active[customers.club_member_status == 'ACTIVE']","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.681145Z","iopub.execute_input":"2022-03-26T12:05:43.681588Z","iopub.status.idle":"2022-03-26T12:05:43.783289Z","shell.execute_reply.started":"2022-03-26T12:05:43.681561Z","shell.execute_reply":"2022-03-26T12:05:43.782151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(active_members.count()) \nprint(customers.Active[customers.Active == 1.0].count())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.784914Z","iopub.execute_input":"2022-03-26T12:05:43.785115Z","iopub.status.idle":"2022-03-26T12:05:43.817954Z","shell.execute_reply.started":"2022-03-26T12:05:43.785092Z","shell.execute_reply":"2022-03-26T12:05:43.81718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### So we have customers that are active club members, but in reality they are not Active or maybey we don't have information about how active are they in real life. Remember that we considered NaN values for Active to be 0. This will help us with machine learning models. We can't drop that column.","metadata":{}},{"cell_type":"markdown","source":"#### Now we start to look at the correlations between tables attributes","metadata":{}},{"cell_type":"code","source":"def create_correlation_heatmap(corr_matrix):\n    plt.figure(figsize=(16, 6))\n\n    mask = np.triu(np.ones_like(corr_matrix)) \n    # the matrix is symmetric so we can view just the under or above triangle relative to main diagonal\n\n    heatmap = sns.heatmap(corr_matrix, mask=mask, vmin=-1, vmax=1, annot=True)\n\n    heatmap.set_title('Correlation for customers', fontdict={'fontsize':18}, pad=16)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.819062Z","iopub.execute_input":"2022-03-26T12:05:43.819341Z","iopub.status.idle":"2022-03-26T12:05:43.824721Z","shell.execute_reply.started":"2022-03-26T12:05:43.819316Z","shell.execute_reply":"2022-03-26T12:05:43.823967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 1. Articles correlation","metadata":{}},{"cell_type":"code","source":"create_correlation_heatmap(articles.corr())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:43.825863Z","iopub.execute_input":"2022-03-26T12:05:43.826064Z","iopub.status.idle":"2022-03-26T12:05:44.478985Z","shell.execute_reply.started":"2022-03-26T12:05:43.826039Z","shell.execute_reply":"2022-03-26T12:05:44.477799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### This heatmap shows that article_id and product_code are 100% related to each other, which seems logical to me. Others, except some *some_name*_no columns, are either negative or close to 0.","metadata":{}},{"cell_type":"markdown","source":"#### 2. Customers correlation","metadata":{}},{"cell_type":"code","source":"customers.replace(['None', 'Regularly', 'Monthly'], [0, 1, 2])                        \ncustomers.replace(['ACTIVE', 'NO INFO', 'PRE-CREATE', 'LEFT CLUB'], [1, 0, 2, 3])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:44.480535Z","iopub.execute_input":"2022-03-26T12:05:44.481566Z","iopub.status.idle":"2022-03-26T12:05:49.823723Z","shell.execute_reply.started":"2022-03-26T12:05:44.481513Z","shell.execute_reply":"2022-03-26T12:05:49.822857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_correlation_heatmap(customers.corr())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:05:49.824967Z","iopub.execute_input":"2022-03-26T12:05:49.825182Z","iopub.status.idle":"2022-03-26T12:05:50.133876Z","shell.execute_reply.started":"2022-03-26T12:05:49.825155Z","shell.execute_reply":"2022-03-26T12:05:50.132762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here we can see that fashion_news_frequency is strongly correlated with FN and Active. It makes sens because, intuitively speaking, the probability that active customers receive news from H&M should be close to 1. Also FN and Active are strongly correlated. Others are very close to 0 or negative.","metadata":{}},{"cell_type":"markdown","source":"#### TODO: Add time series for transactions and some visualization for articles","metadata":{}},{"cell_type":"markdown","source":"#### Save data","metadata":{}},{"cell_type":"code","source":"customers.to_csv('/kaggle/working/customers_data.csv',index=False)\narticles.to_csv('/kaggle/working/articles_data.csv',index=False)\ntransations_articles_grouped.to_csv('/kaggle/working/transactions_data.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T12:36:30.071249Z","iopub.execute_input":"2022-03-26T12:36:30.071544Z","iopub.status.idle":"2022-03-26T12:37:02.962777Z","shell.execute_reply.started":"2022-03-26T12:36:30.071514Z","shell.execute_reply":"2022-03-26T12:37:02.961213Z"},"trusted":true},"execution_count":null,"outputs":[]}]}