{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-29T17:28:17.248743Z","iopub.execute_input":"2022-05-29T17:28:17.250042Z","iopub.status.idle":"2022-05-29T17:28:17.255771Z","shell.execute_reply.started":"2022-05-29T17:28:17.249972Z","shell.execute_reply":"2022-05-29T17:28:17.254368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_feather(\"../input/amexfeather/train_data.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T16:59:32.465021Z","iopub.execute_input":"2022-05-29T16:59:32.465476Z","iopub.status.idle":"2022-05-29T16:59:38.474553Z","shell.execute_reply.started":"2022-05-29T16:59:32.465432Z","shell.execute_reply":"2022-05-29T16:59:38.473331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T16:59:38.476224Z","iopub.execute_input":"2022-05-29T16:59:38.476709Z","iopub.status.idle":"2022-05-29T16:59:38.509910Z","shell.execute_reply.started":"2022-05-29T16:59:38.476660Z","shell.execute_reply":"2022-05-29T16:59:38.508913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-05-29T16:59:38.512128Z","iopub.execute_input":"2022-05-29T16:59:38.512619Z","iopub.status.idle":"2022-05-29T16:59:38.521928Z","shell.execute_reply.started":"2022-05-29T16:59:38.512573Z","shell.execute_reply":"2022-05-29T16:59:38.521251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Datatypes present in training data\n1) We have 1 object type which is customer id\n\n2) We have 1 datetime feature which is S_2\n\n3) We have 11 categorical features\n\n4) We have 178 features which are either float or integer\n\n##### Note : This is not the original dataset. I am using the dataset present in the [link](http://www.kaggle.com/datasets/munumbutt/amexfeather) as the original is quite heavy and is not fitting in the memory allocated in kaggle notebook","metadata":{}},{"cell_type":"code","source":"train_data.describe(include=\"all\",datetime_is_numeric=True).T","metadata":{"execution":{"iopub.status.busy":"2022-05-29T16:59:38.523299Z","iopub.execute_input":"2022-05-29T16:59:38.523722Z","iopub.status.idle":"2022-05-29T17:02:20.689359Z","shell.execute_reply.started":"2022-05-29T16:59:38.523676Z","shell.execute_reply":"2022-05-29T17:02:20.688209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info(max_cols=200, show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:20.690777Z","iopub.execute_input":"2022-05-29T17:02:20.691314Z","iopub.status.idle":"2022-05-29T17:02:26.774884Z","shell.execute_reply.started":"2022-05-29T17:02:20.691267Z","shell.execute_reply":"2022-05-29T17:02:26.773893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:26.776071Z","iopub.execute_input":"2022-05-29T17:02:26.776405Z","iopub.status.idle":"2022-05-29T17:02:32.268392Z","shell.execute_reply.started":"2022-05-29T17:02:26.776374Z","shell.execute_reply":"2022-05-29T17:02:32.267744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set()\nsns.countplot(x=train_data[\"target\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:32.269572Z","iopub.execute_input":"2022-05-29T17:02:32.271954Z","iopub.status.idle":"2022-05-29T17:02:32.976384Z","shell.execute_reply.started":"2022-05-29T17:02:32.271902Z","shell.execute_reply":"2022-05-29T17:02:32.975501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Looks like we have skewed dataset. The number of samples in the positive class is too high than in the negative class","metadata":{}},{"cell_type":"code","source":"temp_df = pd.DataFrame(train_data.target.value_counts() *100/ train_data.shape[0]).reset_index().\\\n            rename(columns={\"index\":\"Target Labels\",\"target\":\"Percentage of Distribution\"})\nVALUES = temp_df[\"Percentage of Distribution\"].values\nLABELS = [\"Non Default\", \"Default\"]\nCOLORS = [\"#F0F8FF\",\"#00FFFF\"]\nfig = go.Figure(data=[go.Pie(labels=LABELS, values=VALUES,marker=dict(colors=COLORS))])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:36:27.179653Z","iopub.execute_input":"2022-05-29T17:36:27.180427Z","iopub.status.idle":"2022-05-29T17:36:27.226341Z","shell.execute_reply.started":"2022-05-29T17:36:27.180388Z","shell.execute_reply":"2022-05-29T17:36:27.225231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We have ***75% positive*** class and approx ***25% negative*** class","metadata":{}},{"cell_type":"code","source":"list_of_features_having_null = [feature for feature in train_data.columns \n                                if train_data[feature].isnull().sum() > 0]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:33.018081Z","iopub.execute_input":"2022-05-29T17:02:33.018790Z","iopub.status.idle":"2022-05-29T17:02:38.471894Z","shell.execute_reply.started":"2022-05-29T17:02:33.018756Z","shell.execute_reply":"2022-05-29T17:02:38.471080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list_of_features_having_null)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:38.473585Z","iopub.execute_input":"2022-05-29T17:02:38.474484Z","iopub.status.idle":"2022-05-29T17:02:38.481561Z","shell.execute_reply.started":"2022-05-29T17:02:38.474430Z","shell.execute_reply":"2022-05-29T17:02:38.480594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### There are **121 columns** having missing values. We need to decide either to impute them or drop them","metadata":{}},{"cell_type":"code","source":"#finding percentage of missing values for each feature\npercent_missing = train_data.isnull().sum() * 100 / len(train_data)\nmissing_value_df = pd.DataFrame({'column_name': train_data.columns,\n                                 'percent_missing': percent_missing}).reset_index().drop(\"index\",axis=1)\n\nmissing_value_df.sort_values('percent_missing',ascending=False)\n\n# missing_value_df.loc[missing_value_df.percent_missing > 50]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:02:38.483068Z","iopub.execute_input":"2022-05-29T17:02:38.483844Z","iopub.status.idle":"2022-05-29T17:02:44.014808Z","shell.execute_reply.started":"2022-05-29T17:02:38.483771Z","shell.execute_reply":"2022-05-29T17:02:44.013580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(missing_value_df,x=\"column_name\",y=\"percent_missing\",\\\n       title=\"Percentage of Missing Values in the columns having Missing values\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:38:33.663664Z","iopub.execute_input":"2022-05-29T17:38:33.664264Z","iopub.status.idle":"2022-05-29T17:38:33.743956Z","shell.execute_reply.started":"2022-05-29T17:38:33.664219Z","shell.execute_reply":"2022-05-29T17:38:33.742662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Many of the columns having almost ***100% missing*** values. We can remove them from the train and test datasets as we don't get any meaningful information if we impute them","metadata":{}},{"cell_type":"code","source":"categorical_features = [feature for feature in train_data.columns if train_data[feature].dtypes == \"category\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:40:03.904897Z","iopub.execute_input":"2022-05-29T17:40:03.905408Z","iopub.status.idle":"2022-05-29T17:40:03.913909Z","shell.execute_reply.started":"2022-05-29T17:40:03.905371Z","shell.execute_reply":"2022-05-29T17:40:03.912533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_features","metadata":{"execution":{"iopub.status.busy":"2022-05-29T19:04:21.215733Z","iopub.execute_input":"2022-05-29T19:04:21.217059Z","iopub.status.idle":"2022-05-29T19:04:21.225209Z","shell.execute_reply.started":"2022-05-29T19:04:21.216984Z","shell.execute_reply":"2022-05-29T19:04:21.224098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datetime_features = train_data[\"S_2\"]\ndatetime_features","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:40:08.094575Z","iopub.execute_input":"2022-05-29T17:40:08.095931Z","iopub.status.idle":"2022-05-29T17:40:08.105609Z","shell.execute_reply.started":"2022-05-29T17:40:08.095879Z","shell.execute_reply":"2022-05-29T17:40:08.104757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"continous_features = [feature for feature in train_data.columns if train_data[feature].dtypes \\\n                      in [\"int16\",\"int32\",\"int64\",\"float16\", \"float32\", \"float64\"]\n                      and feature not in categorical_features and \"S_2\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:40:08.495663Z","iopub.execute_input":"2022-05-29T17:40:08.496968Z","iopub.status.idle":"2022-05-29T17:40:08.506593Z","shell.execute_reply.started":"2022-05-29T17:40:08.496911Z","shell.execute_reply":"2022-05-29T17:40:08.505326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_df = pd.DataFrame(datetime_features)\ntime_df = pd.DataFrame({\"Month of Spending\":time_df[\"S_2\"].dt.month,\n                       \"Year of Spending\" : time_df[\"S_2\"].dt.year })","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:44:12.930280Z","iopub.execute_input":"2022-05-29T18:44:12.931046Z","iopub.status.idle":"2022-05-29T18:44:14.018853Z","shell.execute_reply.started":"2022-05-29T18:44:12.931009Z","shell.execute_reply":"2022-05-29T18:44:14.018121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(time_df.loc[time_df[\"Year of Spending\"] == 2017][\"Month of Spending\"].\\\n             value_counts()).plot(kind=\"bar\",\n                                  figsize=(15,8),\n                                  title=\"Active Number of Customers in the year 2017\",\n                                  xlabel=\"Month in year 2017\",\n                                  ylabel=\"Active Number of Customers\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:54:38.365996Z","iopub.execute_input":"2022-05-29T18:54:38.366526Z","iopub.status.idle":"2022-05-29T18:54:38.752810Z","shell.execute_reply.started":"2022-05-29T18:54:38.366475Z","shell.execute_reply":"2022-05-29T18:54:38.751733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_df.loc[time_df[\"Year of Spending\"] == 2018][\"Month of Spending\"].\\\n        value_counts().plot(kind=\"bar\",\n                            figsize=(15,8),\n                            title=\"Active Number of Customers in the year 2018\",\n                            xlabel=\"Month in year 2018\",\n                            ylabel=\"Active Number of Customers\")","metadata":{"execution":{"iopub.status.busy":"2022-05-29T18:55:17.334723Z","iopub.execute_input":"2022-05-29T18:55:17.335192Z","iopub.status.idle":"2022-05-29T18:55:17.594466Z","shell.execute_reply.started":"2022-05-29T18:55:17.335155Z","shell.execute_reply":"2022-05-29T18:55:17.593327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### From above 2 figures we see that maximum number of customers active in the month of December in 2017 while for 2018 we only have data for first 3 months. So in year 2018 the maximum number of customers active in March month","metadata":{}},{"cell_type":"markdown","source":"## Please do provide your feedback","metadata":{}}]}