{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## EDA on articles.csv","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:17.283821Z","iopub.execute_input":"2022-04-25T13:30:17.284038Z","iopub.status.idle":"2022-04-25T13:30:18.238321Z","shell.execute_reply.started":"2022-04-25T13:30:17.284016Z","shell.execute_reply":"2022-04-25T13:30:18.237807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Loading articles.csv\narticle_df = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\narticle_df.head()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:18.239689Z","iopub.execute_input":"2022-04-25T13:30:18.241731Z","iopub.status.idle":"2022-04-25T13:30:19.370224Z","shell.execute_reply.started":"2022-04-25T13:30:18.241690Z","shell.execute_reply":"2022-04-25T13:30:19.369473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.371071Z","iopub.execute_input":"2022-04-25T13:30:19.371225Z","iopub.status.idle":"2022-04-25T13:30:19.444046Z","shell.execute_reply.started":"2022-04-25T13:30:19.371205Z","shell.execute_reply":"2022-04-25T13:30:19.443144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data type\n## All are nominal data\narticle_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.445186Z","iopub.execute_input":"2022-04-25T13:30:19.445383Z","iopub.status.idle":"2022-04-25T13:30:19.453131Z","shell.execute_reply.started":"2022-04-25T13:30:19.445358Z","shell.execute_reply":"2022-04-25T13:30:19.451933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dataset shape\narticle_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.454738Z","iopub.execute_input":"2022-04-25T13:30:19.454931Z","iopub.status.idle":"2022-04-25T13:30:19.464612Z","shell.execute_reply.started":"2022-04-25T13:30:19.454909Z","shell.execute_reply":"2022-04-25T13:30:19.463773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Seems like each variables \"_no\" or \"_id\" or \"_code\" correspond to the \"_name\"\n## Drop all column with \"_no\", \"_id\", \"_code\" to prevent the machine thinking that \"1\" is more than \"2\", except \"articel_id\"\n\narticle_df_no1 = article_df.iloc[:, 1:]\n\narticle_df_no1.drop(article_df_no1.columns[article_df_no1.columns.str.contains('_no|_code|id')], axis=1, inplace=True)\n\narticle_df2 = pd.concat([article_df_no1, article_df['article_id']], axis =1)\narticle_df2","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.465638Z","iopub.execute_input":"2022-04-25T13:30:19.465803Z","iopub.status.idle":"2022-04-25T13:30:19.536017Z","shell.execute_reply.started":"2022-04-25T13:30:19.465783Z","shell.execute_reply":"2022-04-25T13:30:19.535270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_df2.info()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.537380Z","iopub.execute_input":"2022-04-25T13:30:19.537674Z","iopub.status.idle":"2022-04-25T13:30:19.687272Z","shell.execute_reply.started":"2022-04-25T13:30:19.537628Z","shell.execute_reply":"2022-04-25T13:30:19.686284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check for missing value\n\narticle_df2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.688209Z","iopub.execute_input":"2022-04-25T13:30:19.688816Z","iopub.status.idle":"2022-04-25T13:30:19.744140Z","shell.execute_reply.started":"2022-04-25T13:30:19.688793Z","shell.execute_reply":"2022-04-25T13:30:19.743754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## For the sake of ease of EDA, will drop column \"detail desc\". Other column already give almost the same description already\n## Now, no missing value\n\narticle_df2.drop('detail_desc', axis= 1, inplace= True)\narticle_df2.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.745175Z","iopub.execute_input":"2022-04-25T13:30:19.745653Z","iopub.status.idle":"2022-04-25T13:30:19.780176Z","shell.execute_reply.started":"2022-04-25T13:30:19.745631Z","shell.execute_reply":"2022-04-25T13:30:19.779537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## There are three redundant columns: 'colour_group_name', 'perceived_colour_value_name', 'perceived_colour_master_name'\n## Checking the value of each column to see if they are the same or similar --> may delete if thet are similar\n\nprint(article_df2['colour_group_name'].value_counts())\nprint(article_df2['perceived_colour_value_name'].value_counts())\nprint(article_df2['perceived_colour_master_name'].value_counts())","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:19.781065Z","iopub.execute_input":"2022-04-25T13:30:19.781312Z","iopub.status.idle":"2022-04-25T13:30:19.804238Z","shell.execute_reply.started":"2022-04-25T13:30:19.781291Z","shell.execute_reply":"2022-04-25T13:30:19.803567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## they all explain the color in a similar way \n## possibly when customer search, the result will be 'perceived_colour_value_name' + 'perceived_colour_master_name' = 'colour_group_name\n## then no use for 'colour_group_name'\n\narticle_df2.drop('colour_group_name', axis=1, inplace=True)\narticle_df2.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.805670Z","iopub.execute_input":"2022-04-25T13:30:19.806026Z","iopub.status.idle":"2022-04-25T13:30:19.831973Z","shell.execute_reply.started":"2022-04-25T13:30:19.805990Z","shell.execute_reply":"2022-04-25T13:30:19.831535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## See if \"index_name\" and \"index_group_name\" are redundant\nprint(article_df2['index_name'].value_counts())\nprint(article_df2['index_group_name'].value_counts())\n\n## See if \"department_name\" and \"garment_group_name\" are redundant\nprint(article_df2['department_name'].value_counts())\nprint(article_df2['garment_group_name'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.832798Z","iopub.execute_input":"2022-04-25T13:30:19.833035Z","iopub.status.idle":"2022-04-25T13:30:19.893889Z","shell.execute_reply.started":"2022-04-25T13:30:19.833015Z","shell.execute_reply":"2022-04-25T13:30:19.892971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##'garment_group_name' name things in a weird way --> remove\narticle_df2.drop('garment_group_name', axis=1, inplace=True)\narticle_df2.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.895315Z","iopub.execute_input":"2022-04-25T13:30:19.895566Z","iopub.status.idle":"2022-04-25T13:30:19.924703Z","shell.execute_reply.started":"2022-04-25T13:30:19.895509Z","shell.execute_reply":"2022-04-25T13:30:19.923909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check how many variety of item in each column are there\n\np = 0\nfor col in article_df2.columns:\n    p = article_df2[col].value_counts().count()\n    print(col, ':', p)\n    \n## There are 45875 products in the store, ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:19.928181Z","iopub.execute_input":"2022-04-25T13:30:19.928340Z","iopub.status.idle":"2022-04-25T13:30:20.028559Z","shell.execute_reply.started":"2022-04-25T13:30:19.928320Z","shell.execute_reply":"2022-04-25T13:30:20.027926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check point\n\narticle_df3 = article_df2.copy()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.029627Z","iopub.execute_input":"2022-04-25T13:30:20.031284Z","iopub.status.idle":"2022-04-25T13:30:20.042021Z","shell.execute_reply.started":"2022-04-25T13:30:20.031205Z","shell.execute_reply":"2022-04-25T13:30:20.041137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_df3['prod_name'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.043172Z","iopub.execute_input":"2022-04-25T13:30:20.043417Z","iopub.status.idle":"2022-04-25T13:30:20.092046Z","shell.execute_reply.started":"2022-04-25T13:30:20.043388Z","shell.execute_reply":"2022-04-25T13:30:20.090940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Create function to compute piechart of top 5 most frequent value\ndef piechart(data_set):\n    n = 5\n    tp = data_set.value_counts().sort_values(ascending = False)\n    top = tp[:n].index.tolist()\n    temp = data_set.value_counts().head(n)\n    colors = sns.color_palette('pastel')[0:5]\n    explode = [0.1,0.0,0.01,0.01,0.01]\n    \n    plt.pie(temp, labels = top, colors = colors, explode = explode, autopct='%.0f%%')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.093326Z","iopub.execute_input":"2022-04-25T13:30:20.093559Z","iopub.status.idle":"2022-04-25T13:30:20.100755Z","shell.execute_reply.started":"2022-04-25T13:30:20.093511Z","shell.execute_reply":"2022-04-25T13:30:20.100003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column = article_df3.columns.tolist()\ncolumn","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.102569Z","iopub.execute_input":"2022-04-25T13:30:20.102793Z","iopub.status.idle":"2022-04-25T13:30:20.120542Z","shell.execute_reply.started":"2022-04-25T13:30:20.102760Z","shell.execute_reply":"2022-04-25T13:30:20.119582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##Checking product type and group\nproduct_type_column = ['product_type_name', 'product_group_name']\nfor col in product_type_column:\n    plt.title(col,fontsize=20, pad = 3.0)\n    piechart(article_df3[col])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.121791Z","iopub.execute_input":"2022-04-25T13:30:20.121984Z","iopub.status.idle":"2022-04-25T13:30:20.447312Z","shell.execute_reply.started":"2022-04-25T13:30:20.121958Z","shell.execute_reply":"2022-04-25T13:30:20.446783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check what 'Dress' product_group_name is \ndress = article_df3[article_df3['product_type_name'].str.contains('Dress')]\ndress.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.450309Z","iopub.execute_input":"2022-04-25T13:30:20.452462Z","iopub.status.idle":"2022-04-25T13:30:20.535342Z","shell.execute_reply.started":"2022-04-25T13:30:20.452423Z","shell.execute_reply":"2022-04-25T13:30:20.534844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From what we can see here, 'Garment upper body' constitue almost half of the store with top 3 product types are Sweater, T-Shirt, and Top. 'Trouser' seems to be the most product for Garment Lower body. They also stock alot of 'Dress' even though 'Garment Full Body' contributed only 14% of the total product.","metadata":{}},{"cell_type":"code","source":"## Checking the Design of the poduct\nproduct_type_column = [ 'graphical_appearance_name',\n 'perceived_colour_value_name',\n 'perceived_colour_master_name',]\nfor col in product_type_column:\n    plt.title(col,fontsize=20, pad = 3.0)\n    piechart(article_df3[col])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.536086Z","iopub.execute_input":"2022-04-25T13:30:20.537068Z","iopub.status.idle":"2022-04-25T13:30:20.961190Z","shell.execute_reply.started":"2022-04-25T13:30:20.537010Z","shell.execute_reply":"2022-04-25T13:30:20.959994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The most common theme of the product is 'Dark' with 'Black' and 'Blue' dominating the product line. About 2/3 of the product line has no pattern.","metadata":{}},{"cell_type":"code","source":"## Checking the categories of the poduct\nproduct_type_column = [\n 'index_name',\n 'index_group_name',\n 'section_name']\nfor col in product_type_column:\n    plt.title(col,fontsize=20, pad = 3.0)\n    piechart(article_df3[col])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:20.964582Z","iopub.execute_input":"2022-04-25T13:30:20.964812Z","iopub.status.idle":"2022-04-25T13:30:21.316404Z","shell.execute_reply.started":"2022-04-25T13:30:20.964785Z","shell.execute_reply":"2022-04-25T13:30:21.315642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Women's clothes comprised most of the H&M porduct categories. From what we can see, both Ladieswear and Baby/Children constitue to a little more than 2/3 of the store with Ladieswear at 38%. Most proportion of Baby/children wear seems to be for 'Young Girl' and 'Kids Girl'.","metadata":{}},{"cell_type":"markdown","source":"## EDA on Customer.csv","metadata":{}},{"cell_type":"code","source":"pd.set_option(\"display.max_rows\", None)\ncus_df = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/customers.csv')\ncus_df.head(100)","metadata":{"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:21.317382Z","iopub.execute_input":"2022-04-25T13:30:21.317599Z","iopub.status.idle":"2022-04-25T13:30:26.825549Z","shell.execute_reply.started":"2022-04-25T13:30:21.317572Z","shell.execute_reply":"2022-04-25T13:30:26.824516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Cannot find how to read the postal_code in the given format --> plan to group based on city or country, but no luck trying to read it, so will drop 'postal_code'. \n- Seems like 'FN' = if a customer get Fashion News newsletter,\n    - Check if there is more value than 'Regulary' and 'NONE' -> if only these two then will drop 'fashion_news_frequency' as FN is already binary\n- Have check the value for both 'club_member_status and 'Active' first\n\nfor column explanation\nhttps://www.kaggle.com/c/h-and-m-personalized-fashion-recommendations/discussion/307001","metadata":{"tags":[]}},{"cell_type":"code","source":"cus_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:26.826506Z","iopub.execute_input":"2022-04-25T13:30:26.826726Z","iopub.status.idle":"2022-04-25T13:30:27.410350Z","shell.execute_reply.started":"2022-04-25T13:30:26.826700Z","shell.execute_reply":"2022-04-25T13:30:27.409254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check columns\ncolumn_cus = cus_df.columns\ncolumn_cus","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:27.411373Z","iopub.execute_input":"2022-04-25T13:30:27.411775Z","iopub.status.idle":"2022-04-25T13:30:27.418117Z","shell.execute_reply.started":"2022-04-25T13:30:27.411751Z","shell.execute_reply":"2022-04-25T13:30:27.417229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the sshape\ncus_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:27.419316Z","iopub.execute_input":"2022-04-25T13:30:27.419672Z","iopub.status.idle":"2022-04-25T13:30:27.431018Z","shell.execute_reply.started":"2022-04-25T13:30:27.419640Z","shell.execute_reply":"2022-04-25T13:30:27.430432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## only 'postal_code' can be dropped\ncus_df2 = cus_df.copy()\ncus_df2.drop('postal_code', axis= 1, inplace= True)\ncus_df2.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:27.431980Z","iopub.execute_input":"2022-04-25T13:30:27.432849Z","iopub.status.idle":"2022-04-25T13:30:27.577174Z","shell.execute_reply.started":"2022-04-25T13:30:27.432814Z","shell.execute_reply":"2022-04-25T13:30:27.576691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check for missing value\ncus_df2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:27.578279Z","iopub.execute_input":"2022-04-25T13:30:27.579175Z","iopub.status.idle":"2022-04-25T13:30:28.007224Z","shell.execute_reply.started":"2022-04-25T13:30:27.579119Z","shell.execute_reply":"2022-04-25T13:30:28.006420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cus_df2['FN'].value_counts())\nprint(cus_df2['Active'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.008368Z","iopub.execute_input":"2022-04-25T13:30:28.008656Z","iopub.status.idle":"2022-04-25T13:30:28.046562Z","shell.execute_reply.started":"2022-04-25T13:30:28.008620Z","shell.execute_reply":"2022-04-25T13:30:28.045649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dealing with missing value\ncus_df2['FN'] = cus_df2['FN'].fillna(0)\ncus_df2['Active'] = cus_df2['Active'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.048018Z","iopub.execute_input":"2022-04-25T13:30:28.048238Z","iopub.status.idle":"2022-04-25T13:30:28.073031Z","shell.execute_reply.started":"2022-04-25T13:30:28.048209Z","shell.execute_reply":"2022-04-25T13:30:28.072159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cus_df2['club_member_status'].value_counts())\nprint('number of missing value =', cus_df2['club_member_status'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.075356Z","iopub.execute_input":"2022-04-25T13:30:28.075676Z","iopub.status.idle":"2022-04-25T13:30:28.421696Z","shell.execute_reply.started":"2022-04-25T13:30:28.075637Z","shell.execute_reply":"2022-04-25T13:30:28.420217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Use mode to replace missing value\ncus_df2['club_member_status'] = cus_df2['club_member_status'].fillna(cus_df2['club_member_status'].mode()[0])\nprint(cus_df2['club_member_status'].value_counts())\nprint('number of missing value =', cus_df2['club_member_status'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.422801Z","iopub.execute_input":"2022-04-25T13:30:28.423015Z","iopub.status.idle":"2022-04-25T13:30:28.784446Z","shell.execute_reply.started":"2022-04-25T13:30:28.422983Z","shell.execute_reply":"2022-04-25T13:30:28.783765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check for missing value and value type\nprint(cus_df2['fashion_news_frequency'].value_counts())\nprint('number of missing value =', cus_df2['fashion_news_frequency'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.785286Z","iopub.execute_input":"2022-04-25T13:30:28.785442Z","iopub.status.idle":"2022-04-25T13:30:28.914176Z","shell.execute_reply.started":"2022-04-25T13:30:28.785420Z","shell.execute_reply":"2022-04-25T13:30:28.913465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## merge Redundant value NONE and None into 'NONE'\ncus_df2['fashion_news_frequency'] = cus_df2['fashion_news_frequency'].replace('None', 'NONE')\n\n## replacing missing value with mode\ncus_df2['fashion_news_frequency'] = cus_df2['fashion_news_frequency'].fillna(cus_df2['fashion_news_frequency'].mode()[0])\n\n#Recheck\nprint(cus_df2['fashion_news_frequency'].value_counts())\nprint('number of missing value =', cus_df2['fashion_news_frequency'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:28.915263Z","iopub.execute_input":"2022-04-25T13:30:28.915475Z","iopub.status.idle":"2022-04-25T13:30:29.251836Z","shell.execute_reply.started":"2022-04-25T13:30:28.915447Z","shell.execute_reply":"2022-04-25T13:30:29.250968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Checking range of 'age'\nmax_age = max(cus_df2['age'])\nmin_age = min(cus_df2['age'])\nprint('max age is', max_age)\nprint('minimun age is', min_age)\nprint('range of \"age\" is', max_age - min_age)\n## Check for missing value in 'age'\nprint('number of missing value =', cus_df2['age'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:29.252794Z","iopub.execute_input":"2022-04-25T13:30:29.253128Z","iopub.status.idle":"2022-04-25T13:30:29.484328Z","shell.execute_reply.started":"2022-04-25T13:30:29.253106Z","shell.execute_reply":"2022-04-25T13:30:29.483432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## minimum age of 16 make sense, but 90 years old might not make sense\n## Ploting histogram: bimodal and skew to the left\n\nsns.histplot(cus_df2['age'])","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:29.485473Z","iopub.execute_input":"2022-04-25T13:30:29.485727Z","iopub.status.idle":"2022-04-25T13:30:31.317305Z","shell.execute_reply.started":"2022-04-25T13:30:29.485693Z","shell.execute_reply":"2022-04-25T13:30:31.316511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Seeing the percentage of missing value in 'age' to the total number of row to see if it is possible to delete the row that have missing value in age\n## only 1.15% --> delete the column\nper = 15861/1371979 * 100\nper","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:31.318377Z","iopub.execute_input":"2022-04-25T13:30:31.318604Z","iopub.status.idle":"2022-04-25T13:30:31.324914Z","shell.execute_reply.started":"2022-04-25T13:30:31.318575Z","shell.execute_reply":"2022-04-25T13:30:31.324015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cus_df2.dropna(axis = 0, inplace=True)\ncus_df2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:31.325988Z","iopub.execute_input":"2022-04-25T13:30:31.326211Z","iopub.status.idle":"2022-04-25T13:30:31.729561Z","shell.execute_reply.started":"2022-04-25T13:30:31.326182Z","shell.execute_reply":"2022-04-25T13:30:31.728835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plotting boxplot\nage = cus_df2['age']\n\nplt.boxplot(age, vert=False)\nplt.title(\"Detecting outliers using Boxplot\")\nplt.xlabel('sample')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:31.730497Z","iopub.execute_input":"2022-04-25T13:30:31.730714Z","iopub.status.idle":"2022-04-25T13:30:31.878925Z","shell.execute_reply.started":"2022-04-25T13:30:31.730688Z","shell.execute_reply":"2022-04-25T13:30:31.878161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finding outlier, upper bound, and lower bound\noutliers = []\ndef detect_outliers_iqr(data):\n    data = sorted(data)\n    q1 = np.percentile(data, 25)\n    q3 = np.percentile(data, 75)\n    \n    IQR = q3-q1\n    lwr_bound = q1-(1.5*IQR)\n    upr_bound = q3+(1.5*IQR)\n    print(\"Upper bound is\", upr_bound)\n    print(\"Lower bound is\", lwr_bound)\n    \n    for i in data:\n        if (i<lwr_bound or i>upr_bound):\n            outliers.append(i)\n    return outliers\nage_outliers = detect_outliers_iqr(age)\nprint(\"Outliers from IQR method: \", age_outliers)\nprint(\"Min value of outliers from IQR method: \", min(age_outliers))","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:31.879922Z","iopub.execute_input":"2022-04-25T13:30:31.880147Z","iopub.status.idle":"2022-04-25T13:30:34.016344Z","shell.execute_reply.started":"2022-04-25T13:30:31.880116Z","shell.execute_reply":"2022-04-25T13:30:34.015400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Putting a cap on the maximum age --> converting outlier age to age at 95th percentile\n## Finding 95th percentile first\nninety_fifth_percentile = np.percentile(age, 95)\nprint(\"95th percentile is\", ninety_fifth_percentile, \"years old\")\n\ncus_df2['age'] = np.where((cus_df2['age']> 62), 62 ,cus_df2['age'])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:34.021285Z","iopub.execute_input":"2022-04-25T13:30:34.021555Z","iopub.status.idle":"2022-04-25T13:30:34.044719Z","shell.execute_reply.started":"2022-04-25T13:30:34.021513Z","shell.execute_reply":"2022-04-25T13:30:34.043735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Check if there is still outliers\nplt.boxplot(cus_df2['age'], vert=False)\nplt.title(\"Detecting outliers using Boxplot\")\nplt.xlabel('sample')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:34.045987Z","iopub.execute_input":"2022-04-25T13:30:34.046216Z","iopub.status.idle":"2022-04-25T13:30:34.217510Z","shell.execute_reply.started":"2022-04-25T13:30:34.046187Z","shell.execute_reply":"2022-04-25T13:30:34.216649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Checkpoint \ncus_df3 = cus_df2.copy()","metadata":{"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:34.218686Z","iopub.execute_input":"2022-04-25T13:30:34.219667Z","iopub.status.idle":"2022-04-25T13:30:34.272549Z","shell.execute_reply.started":"2022-04-25T13:30:34.219634Z","shell.execute_reply":"2022-04-25T13:30:34.271793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cus_df3.head(10)","metadata":{"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-04-25T13:30:34.273468Z","iopub.execute_input":"2022-04-25T13:30:34.274566Z","iopub.status.idle":"2022-04-25T13:30:34.287615Z","shell.execute_reply.started":"2022-04-25T13:30:34.274489Z","shell.execute_reply":"2022-04-25T13:30:34.286971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Seeing proportion of each chart\ndef piechart2(data_set):\n    top = data_set.value_counts().index.tolist()\n    temp = data_set.value_counts()\n    colors = sns.color_palette('bright')[0:5]\n    \n    \n    plt.pie(temp, labels = top, colors = colors, autopct='%.0f%%')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:41:29.586225Z","iopub.execute_input":"2022-04-25T13:41:29.587112Z","iopub.status.idle":"2022-04-25T13:41:29.593332Z","shell.execute_reply.started":"2022-04-25T13:41:29.587078Z","shell.execute_reply":"2022-04-25T13:41:29.592579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FN = cus_df3['FN']\nActive = cus_df3['Active']\nclub_member = cus_df3['club_member_status']\nfashion_news = cus_df3['fashion_news_frequency']\n\nfont_size = 10\n\nfig, axs = plt.subplots(2,2)\n\n\nlabels = FN.value_counts().index.tolist()\ndata = FN.value_counts()\naxs[0,0].pie(data, labels=labels, autopct='%1.1f%%', shadow=True, radius=5)\nplt.title('FN',fontsize=font_size, pad = 1.0)\n\n\nlabels = Active.value_counts().index.tolist()\ndata = Active.value_counts()\naxs[0,1].pie(data, labels=labels, autopct='%.0f%%', shadow=True, radius=5)\n\n          \nlabels = club_member.value_counts().index.tolist()\ndata = club_member.value_counts()\naxs[1, 0].pie(data, labels=labels, autopct='%.0f%%', shadow=True, radius=5)\n\n          \nlabels = fashion_news.value_counts().index.tolist()\ndata = fashion_news.value_counts()\naxs[1, 1].pie(data, labels=labels, autopct='%.0f%%', shadow=True, radius=5)\n\n\nplt.subplots_adjust(wspace=5, hspace=5)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:30:34.288816Z","iopub.execute_input":"2022-04-25T13:30:34.289034Z","iopub.status.idle":"2022-04-25T13:30:34.978009Z","shell.execute_reply.started":"2022-04-25T13:30:34.289006Z","shell.execute_reply":"2022-04-25T13:30:34.977395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5,5), dpi=100)\n#2 rows 2 columns\n\n#first row, first column\nax1 = plt.subplot2grid((2,2),(0,0))\npiechart2(FN)\nplt.title('FN')\n\n#first row sec column\nax1 = plt.subplot2grid((2,2), (0, 1))\npiechart2(Active)\nplt.title('Active')\n\n#Second row first column\nax1 = plt.subplot2grid((2,2), (1, 0))\npiechart2(club_member)\nplt.title('club_member')\n\n#second row second column\nax1 = plt.subplot2grid((2,2), (1, 1))\npiechart2(fashion_news)\nplt.title('fashion_news')","metadata":{"execution":{"iopub.status.busy":"2022-04-25T13:41:54.256042Z","iopub.execute_input":"2022-04-25T13:41:54.256313Z","iopub.status.idle":"2022-04-25T13:41:55.385950Z","shell.execute_reply.started":"2022-04-25T13:41:54.256285Z","shell.execute_reply":"2022-04-25T13:41:55.385113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}