{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:06:50.986892Z","iopub.execute_input":"2022-06-30T14:06:50.987868Z","iopub.status.idle":"2022-06-30T14:06:51.025517Z","shell.execute_reply.started":"2022-06-30T14:06:50.987771Z","shell.execute_reply":"2022-06-30T14:06:51.024625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imblearn\n!pip install LightGBM\n!pip install XGBosst\n!pip install CatBoost","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:06:51.116906Z","iopub.execute_input":"2022-06-30T14:06:51.117206Z","iopub.status.idle":"2022-06-30T14:07:22.870482Z","shell.execute_reply.started":"2022-06-30T14:06:51.117179Z","shell.execute_reply":"2022-06-30T14:07:22.869314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Library\nimport gc\nfrom itertools import product\n\n# Graphic Components\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Statistical Inference Analyis\nimport statsmodels.api as sm\nimport scipy.stats as stats\n\n# Data Precessing \nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.impute import SimpleImputer\n\n# Model\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\nimport lightgbm as lgbm\nimport xgboost as xgb \nimport catboost\n\n# Feature Selection/ Model Optimization\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.ensemble import BaggingClassifier\n\n# Validation \nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import plot_roc_curve\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:22.872656Z","iopub.execute_input":"2022-06-30T14:07:22.873156Z","iopub.status.idle":"2022-06-30T14:07:26.850190Z","shell.execute_reply.started":"2022-06-30T14:07:22.873109Z","shell.execute_reply":"2022-06-30T14:07:26.849317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df = pd.read_feather('../input/amexfeather/train_data.ftr')\nTrain_df.head()\n\n#Test_df = pd.read_feather('../input/amexfeather/test_data.ftr')\n#Test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:26.851540Z","iopub.execute_input":"2022-06-30T14:07:26.852075Z","iopub.status.idle":"2022-06-30T14:07:48.763335Z","shell.execute_reply.started":"2022-06-30T14:07:26.852031Z","shell.execute_reply":"2022-06-30T14:07:48.762436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:48.765737Z","iopub.execute_input":"2022-06-30T14:07:48.766139Z","iopub.status.idle":"2022-06-30T14:07:48.771975Z","shell.execute_reply.started":"2022-06-30T14:07:48.766101Z","shell.execute_reply":"2022-06-30T14:07:48.771055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As all features of the dataset are masked, we cannot rely on domain knowledge nor common sense to get a general picture on the dataset. Let us make some wild guesses by examining the data type of each feature.","metadata":{}},{"cell_type":"code","source":"Train_df.dtypes.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:48.773621Z","iopub.execute_input":"2022-06-30T14:07:48.774003Z","iopub.status.idle":"2022-06-30T14:07:48.790057Z","shell.execute_reply.started":"2022-06-30T14:07:48.773963Z","shell.execute_reply":"2022-06-30T14:07:48.789223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that the customer ID is duplicated and there is a datetime column (S_2). 11 features are categorial data, and the remaining are all numeric features. Although the information is still very limited, the above result implies that the dataset should be very likely a transaction data.\n\nGiven this observation, there is two ways ahead. The first is time-series analysis on the features, and the second should be taking the latest snapshot with a lower scale. However, the second way is quite dangerous as it may overlook some time-serial features and distort the features importance. \n\nHowever, under the limited capacity of Kaggle Environment, we can only take the first way. Let us return if any chances!","metadata":{}},{"cell_type":"code","source":"Train_df = Train_df.groupby('customer_ID').tail(1)\nTrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:48.791456Z","iopub.execute_input":"2022-06-30T14:07:48.792062Z","iopub.status.idle":"2022-06-30T14:07:51.088496Z","shell.execute_reply.started":"2022-06-30T14:07:48.792022Z","shell.execute_reply":"2022-06-30T14:07:51.087457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset has been scaled down 10x when taking the latest records only.","metadata":{}},{"cell_type":"code","source":"Null_Check = pd.DataFrame({'Columns':Train_df.columns,\n                           'Null Ratio':Train_df.isna().sum().values / len(Train_df)}).sort_values(by = ['Null Ratio'], ascending = False)\nNull_Check.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:51.089765Z","iopub.execute_input":"2022-06-30T14:07:51.090159Z","iopub.status.idle":"2022-06-30T14:07:51.652612Z","shell.execute_reply.started":"2022-06-30T14:07:51.090121Z","shell.execute_reply":"2022-06-30T14:07:51.651588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(12, 6))\nsns.histplot(Null_Check['Null Ratio'])\nplt.title('Histogram of Null Ratio')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:51.657377Z","iopub.execute_input":"2022-06-30T14:07:51.657845Z","iopub.status.idle":"2022-06-30T14:07:52.395457Z","shell.execute_reply.started":"2022-06-30T14:07:51.657806Z","shell.execute_reply":"2022-06-30T14:07:52.394615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Null Ratio Median:', Null_Check['Null Ratio'].quantile(.5))\nprint('Null Ratio Average:', Null_Check['Null Ratio'].mean())","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.397112Z","iopub.execute_input":"2022-06-30T14:07:52.397673Z","iopub.status.idle":"2022-06-30T14:07:52.407617Z","shell.execute_reply.started":"2022-06-30T14:07:52.397632Z","shell.execute_reply":"2022-06-30T14:07:52.406606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in np.linspace(0,1, 11).round(1):\n    print(i, len(Null_Check[Null_Check['Null Ratio'] > i]))\n    \nDrop_Columns = Null_Check[Null_Check['Null Ratio'] > 0.7]['Columns']\nDrop_Columns","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.412194Z","iopub.execute_input":"2022-06-30T14:07:52.412676Z","iopub.status.idle":"2022-06-30T14:07:52.428315Z","shell.execute_reply.started":"2022-06-30T14:07:52.412626Z","shell.execute_reply":"2022-06-30T14:07:52.427309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The histogram shows that most of the most columns are fine (close to left tail 0% null ratio), except some outliners. Let us remove these cases as they should have very limited importance to our models.","metadata":{}},{"cell_type":"code","source":"Train_df = Train_df.drop(columns = Null_Check[Null_Check['Null Ratio'] > 0.7]['Columns'])\nTrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.429814Z","iopub.execute_input":"2022-06-30T14:07:52.430370Z","iopub.status.idle":"2022-06-30T14:07:52.700396Z","shell.execute_reply.started":"2022-06-30T14:07:52.430335Z","shell.execute_reply":"2022-06-30T14:07:52.699041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.702842Z","iopub.execute_input":"2022-06-30T14:07:52.703392Z","iopub.status.idle":"2022-06-30T14:07:52.713749Z","shell.execute_reply.started":"2022-06-30T14:07:52.703351Z","shell.execute_reply":"2022-06-30T14:07:52.712823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The distribution of target is 3:1. We will keep this in mind and handle it in the later section.","metadata":{}},{"cell_type":"code","source":"del Null_Check\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.715353Z","iopub.execute_input":"2022-06-30T14:07:52.716078Z","iopub.status.idle":"2022-06-30T14:07:52.877520Z","shell.execute_reply.started":"2022-06-30T14:07:52.716038Z","shell.execute_reply":"2022-06-30T14:07:52.876550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploration Data Analysis","metadata":{}},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(15, 15))\nsns.heatmap(Train_df.corr())\nplt.title('Correlation Matrix')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:07:52.879347Z","iopub.execute_input":"2022-06-30T14:07:52.880054Z","iopub.status.idle":"2022-06-30T14:08:21.671344Z","shell.execute_reply.started":"2022-06-30T14:07:52.880013Z","shell.execute_reply":"2022-06-30T14:08:21.670488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(15, 15))\nsns.heatmap(Train_df[[i for i in Train_df.columns if 'S' in i]].corr(), annot = True, fmt='.1f')\nplt.title('Correlation Matrix - S Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:21.672363Z","iopub.execute_input":"2022-06-30T14:08:21.672726Z","iopub.status.idle":"2022-06-30T14:08:23.970321Z","shell.execute_reply.started":"2022-06-30T14:08:21.672692Z","shell.execute_reply":"2022-06-30T14:08:23.969267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(15, 15))\nsns.heatmap(Train_df[[i for i in Train_df.columns if 'R' in i]].corr(), annot = True, fmt='.1f')\nplt.title('Correlation Matrix - R Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:23.971973Z","iopub.execute_input":"2022-06-30T14:08:23.972378Z","iopub.status.idle":"2022-06-30T14:08:27.568580Z","shell.execute_reply.started":"2022-06-30T14:08:23.972336Z","shell.execute_reply":"2022-06-30T14:08:27.567631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(10, 10))\nsns.heatmap(Train_df[[i for i in Train_df.columns if 'P' in i]].corr(), annot = True, fmt='.1f')\nplt.title('Correlation Matrix - P Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:27.570326Z","iopub.execute_input":"2022-06-30T14:08:27.570748Z","iopub.status.idle":"2022-06-30T14:08:27.838296Z","shell.execute_reply.started":"2022-06-30T14:08:27.570707Z","shell.execute_reply":"2022-06-30T14:08:27.837430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(25, 25))\nsns.heatmap(Train_df[[i for i in Train_df.columns if 'B' in i]].corr(), annot = True, fmt='.1f')\nplt.title('Correlation Matrix - B Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:27.839801Z","iopub.execute_input":"2022-06-30T14:08:27.840162Z","iopub.status.idle":"2022-06-30T14:08:33.351502Z","shell.execute_reply.started":"2022-06-30T14:08:27.840126Z","shell.execute_reply":"2022-06-30T14:08:33.350599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(20, 20))\nsns.heatmap(Train_df[[i for i in Train_df.columns if 'D' in i]].corr())\nplt.title('Correlation Matrix - D Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:33.353046Z","iopub.execute_input":"2022-06-30T14:08:33.353448Z","iopub.status.idle":"2022-06-30T14:08:41.587352Z","shell.execute_reply.started":"2022-06-30T14:08:33.353410Z","shell.execute_reply":"2022-06-30T14:08:41.586379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above heatmaps, we found a lot collinear variables. These variables will cause us not only the problem of dimensionality curse but also noises during computatuion, which undermines our statistical inference thus model performances. Let us do a dimensional reduction by the category.","metadata":{}},{"cell_type":"markdown","source":"# PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.decomposition import FactorAnalysis\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\n\nimport statsmodels.api as sm","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:41.588668Z","iopub.execute_input":"2022-06-30T14:08:41.589829Z","iopub.status.idle":"2022-06-30T14:08:41.595171Z","shell.execute_reply.started":"2022-06-30T14:08:41.589787Z","shell.execute_reply":"2022-06-30T14:08:41.594067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare for a PCA \nMaster_df = Train_df[['customer_ID','target']].reset_index(drop = True)\n\n# Categorial\nPCA_Cat = Train_df.select_dtypes(include='category').reset_index(drop = True)\n\nfor i in PCA_Cat.columns:\n    PCA_Cat[i].fillna(PCA_Cat[i].quantile(.5), inplace = True)\n    \nPCA_Cat = pd.get_dummies(PCA_Cat, drop_first= True)\n\n# Numeric and Normalize\nPCA_Numeric = Train_df.select_dtypes(include=['float16']).reset_index(drop = True)\n\nfor i in PCA_Numeric.columns:\n    PCA_Numeric[i] = PCA_Numeric[i].astype('float64')\n    PCA_Numeric[i] = PCA_Numeric[i].fillna(PCA_Numeric[i].mean())\n\nPCA_Numeric = pd.DataFrame(StandardScaler().fit_transform(PCA_Numeric), columns = PCA_Numeric.columns)\n    \n# Concat\nPCA_df = pd.concat([PCA_Cat, PCA_Numeric], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:41.596785Z","iopub.execute_input":"2022-06-30T14:08:41.597436Z","iopub.status.idle":"2022-06-30T14:08:48.166980Z","shell.execute_reply.started":"2022-06-30T14:08:41.597397Z","shell.execute_reply":"2022-06-30T14:08:48.166059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PCA_Model = PCA(n_components=3, random_state=0)\n\nfor cat in ['S','R','B','P','D']:\n    \n    Temp = pd.DataFrame(PCA_Model.fit_transform(PCA_df[[i for i in PCA_df.columns if cat in i]]))\n    Temp.columns = [cat +'_'+ str(name) for name in Temp.columns]\n    Master_df = pd.concat([Master_df, Temp], axis = 1)\n    \nMaster_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:08:48.168500Z","iopub.execute_input":"2022-06-30T14:08:48.168910Z","iopub.status.idle":"2022-06-30T14:09:00.965345Z","shell.execute_reply.started":"2022-06-30T14:08:48.168871Z","shell.execute_reply":"2022-06-30T14:09:00.964382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(15, 15))\nsns.heatmap(Master_df.iloc[:, 2:].corr(),annot = True, fmt='.1f')\nplt.title('Correlation Matrix - PCA')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:00.967090Z","iopub.execute_input":"2022-06-30T14:09:00.967422Z","iopub.status.idle":"2022-06-30T14:09:02.269209Z","shell.execute_reply.started":"2022-06-30T14:09:00.967395Z","shell.execute_reply":"2022-06-30T14:09:02.268137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have reduced into only 10 variables and the problem multicollinearity is now more improved, where the max R coefficient is around .7. However, it still exceeds the conventional threshold of collinear where the absolute value of R coefficient is equal or higher than .6. It seems that the variables categoriztion (R, S, B, D, P) is not as independent as we thought. There are some interwining impacts among these categories.\n\nHere are some observations:\n1. S, R features are quite independent and non-collinear.\n2. D, P and B are still quite collinear (D_0 and B_0 and P_0).","metadata":{}},{"cell_type":"code","source":"import statsmodels.api as sm\nRegression = sm.add_constant(Master_df.iloc[:,2:])\nlogit_mod = sm.Logit(Master_df['target'],Regression)\nlogit_res = logit_mod.fit()\nprint(logit_res.summary())","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:02.270788Z","iopub.execute_input":"2022-06-30T14:09:02.271169Z","iopub.status.idle":"2022-06-30T14:09:04.010135Z","shell.execute_reply.started":"2022-06-30T14:09:02.271128Z","shell.execute_reply":"2022-06-30T14:09:04.009277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the variables are signficant after PCA. And we observe that the importance of Spending variables is quite light. The most significant variables are payment related. Let us have a try to do a PCA in general instead.","metadata":{}},{"cell_type":"code","source":"# Scree \nPCA_Model = PCA(n_components=10, random_state=0)\nPCA_Model.fit(PCA_df)\n\nplt.figure(figsize=(12, 7))\nplt.plot(PCA_Model.explained_variance_ratio_)\nplt.title('Scree Plot')\nplt.xlabel('Principal Component')\nplt.ylabel('Variance Explained')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:04.011745Z","iopub.execute_input":"2022-06-30T14:09:04.012415Z","iopub.status.idle":"2022-06-30T14:09:13.024283Z","shell.execute_reply.started":"2022-06-30T14:09:04.012371Z","shell.execute_reply":"2022-06-30T14:09:13.023476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The optimal number of compontents should be around 4-6. Let us use 6 componentents for our models.","metadata":{}},{"cell_type":"code","source":"PCA_Model = PCA(n_components=6, random_state=0)\nTemp = pd.DataFrame(PCA_Model.fit_transform(PCA_df))\nMaster_df = pd.concat([Master_df.iloc[:, :2], Temp], axis = 1)\nMaster_df","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:13.025486Z","iopub.execute_input":"2022-06-30T14:09:13.026832Z","iopub.status.idle":"2022-06-30T14:09:21.564795Z","shell.execute_reply.started":"2022-06-30T14:09:13.026789Z","shell.execute_reply":"2022-06-30T14:09:21.563396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nplt.figure(figsize=(15, 15))\nsns.heatmap(Master_df.iloc[:, 2:].corr(),annot = True, fmt='.1f')\nplt.title('Correlation Matrix - PCA')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:21.566977Z","iopub.execute_input":"2022-06-30T14:09:21.567529Z","iopub.status.idle":"2022-06-30T14:09:22.029282Z","shell.execute_reply.started":"2022-06-30T14:09:21.567486Z","shell.execute_reply":"2022-06-30T14:09:22.028365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statsmodels.api as sm\nRegression = sm.add_constant(Master_df.iloc[:,2:])\nlogit_mod = sm.Logit(Master_df['target'],Regression)\nlogit_res = logit_mod.fit()\nprint(logit_res.summary())","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:22.035179Z","iopub.execute_input":"2022-06-30T14:09:22.035501Z","iopub.status.idle":"2022-06-30T14:09:23.253635Z","shell.execute_reply.started":"2022-06-30T14:09:22.035473Z","shell.execute_reply":"2022-06-30T14:09:23.252566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After a complete PCA, all variables are now significant (p<0.05). The pseudo R-squ (0.5459) is just slighly lower than the model with the above with collinear. The next step will be building a data model on the PCA data.","metadata":{}},{"cell_type":"markdown","source":"# Data Models","metadata":{}},{"cell_type":"markdown","source":"As mentioned above that the target ratio is 3:1 which may cause bias on our models, let us use SMOTE oversampling method to make create a balance sample.","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE \nsm = SMOTE(random_state=42)\nX, y = sm.fit_resample(Master_df.iloc[:,2:], Master_df['target'])","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:23.258241Z","iopub.execute_input":"2022-06-30T14:09:23.260671Z","iopub.status.idle":"2022-06-30T14:09:27.905366Z","shell.execute_reply.started":"2022-06-30T14:09:23.260618Z","shell.execute_reply":"2022-06-30T14:09:27.904457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:27.906710Z","iopub.execute_input":"2022-06-30T14:09:27.907101Z","iopub.status.idle":"2022-06-30T14:09:27.917288Z","shell.execute_reply.started":"2022-06-30T14:09:27.907061Z","shell.execute_reply":"2022-06-30T14:09:27.916435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Test Split\ntrain_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:27.918553Z","iopub.execute_input":"2022-06-30T14:09:27.919334Z","iopub.status.idle":"2022-06-30T14:09:28.020866Z","shell.execute_reply.started":"2022-06-30T14:09:27.919293Z","shell.execute_reply":"2022-06-30T14:09:28.019946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build a DataFrame To Score the Performance\n\nPerformance_df = pd.DataFrame(columns = ['Model', 'Feature Selection', 'Accuracy', 'Log Loss', 'ROC', 'Amex Metric'])\nPerformance_df","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:28.022370Z","iopub.execute_input":"2022-06-30T14:09:28.022795Z","iopub.status.idle":"2022-06-30T14:09:28.036343Z","shell.execute_reply.started":"2022-06-30T14:09:28.022753Z","shell.execute_reply":"2022-06-30T14:09:28.035351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:28.037941Z","iopub.execute_input":"2022-06-30T14:09:28.038940Z","iopub.status.idle":"2022-06-30T14:09:28.053430Z","shell.execute_reply.started":"2022-06-30T14:09:28.038900Z","shell.execute_reply":"2022-06-30T14:09:28.052607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logit Regression","metadata":{}},{"cell_type":"code","source":"# Randomized Cross Validation for Hyperparameters Tuning\n\ndef Logit_Randomize_CV(X,y, parameters, k = -1):\n    \n    # Create Parameter Combination\n    keys, values = zip(*parameters.items())\n    result = [dict(zip(keys, p)) for p in product(*values)]\n    \n    if k != -1:\n\n        result = np.random.choice(result, k, replace = False)\n    \n    best_score = -1\n    best_parameter = {}\n    best_model = None\n    \n    # Train Test Split\n    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    for i in result:\n        \n        Logit = LogisticRegression(**i)\n        Logit.fit(train_X, train_y)\n        \n        pred = Logit.predict(test_X)\n        amex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\n        amex_prediction = pd.DataFrame({'prediction': Logit.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\n        score = amex_metric(amex_actual,amex_prediction)\n        \n        if score > best_score:\n            best_score = score\n            best_parameter = i\n            best_model = Logit\n        \n        print(f'{i}: {score}')\n        \n    print(f'Best Parameters - {best_parameter}: {best_score}')\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:28.054758Z","iopub.execute_input":"2022-06-30T14:09:28.055211Z","iopub.status.idle":"2022-06-30T14:09:28.067552Z","shell.execute_reply.started":"2022-06-30T14:09:28.055174Z","shell.execute_reply":"2022-06-30T14:09:28.066736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pools of Parameters\nrandom_parameters = {'solver': ['newton-cg', 'lbfgs', 'liblinear'],\n                     'C' :[100, 10, 1.0, 0.1, 0.01]\n                    }\n\nLogit = Logit_Randomize_CV(train_X,train_y, random_parameters)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:28.070515Z","iopub.execute_input":"2022-06-30T14:09:28.070868Z","iopub.status.idle":"2022-06-30T14:09:49.322231Z","shell.execute_reply.started":"2022-06-30T14:09:28.070843Z","shell.execute_reply":"2022-06-30T14:09:49.321251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = Logit.predict(test_X)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame({'prediction': Logit.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['Logit', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred), amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:49.323516Z","iopub.execute_input":"2022-06-30T14:09:49.324158Z","iopub.status.idle":"2022-06-30T14:09:50.388362Z","shell.execute_reply.started":"2022-06-30T14:09:49.324107Z","shell.execute_reply":"2022-06-30T14:09:50.387309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Support Vector","metadata":{}},{"cell_type":"code","source":"def SVM_Bagging_Randomize_CV(X,y, parameters, k = -1):\n    \n    # Create Parameter Combination\n    keys, values = zip(*parameters.items())\n    result = [dict(zip(keys, p)) for p in product(*values)]\n    \n    if k != -1:\n\n        result = np.random.choice(result, k, replace = False)\n        \n    best_score = -1\n    best_parameter = {}\n    best_model = None\n    \n    # Train Test Split\n    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    for i in result:\n        \n        Support_Vector = BaggingClassifier(base_estimator=SVC(**i), \n                                           n_estimators=64,\n                                           max_samples = 128, \n                                           random_state=0,\n                                           n_jobs = -1)\n\n        Support_Vector.fit(train_X, train_y)\n        \n        pred = Support_Vector.predict(test_X)\n        amex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\n        amex_prediction = pd.DataFrame({'prediction': Support_Vector.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\n        score = amex_metric(amex_actual,amex_prediction)\n        \n        if score > best_score:\n            best_score = score\n            best_parameter = i\n            best_model = Support_Vector\n        \n        print(f'{i}: {score}')\n        \n    print(f'Best Parameters - {best_parameter}: {best_score}')\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:50.390103Z","iopub.execute_input":"2022-06-30T14:09:50.390511Z","iopub.status.idle":"2022-06-30T14:09:50.401673Z","shell.execute_reply.started":"2022-06-30T14:09:50.390470Z","shell.execute_reply":"2022-06-30T14:09:50.400767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pools of Parameters   \n\nrandom_parameters = {'C': [1, 10, 100], \n                     'gamma': [0.1,0.01, 0.001],\n                     'kernel': ['linear', 'poly', 'rbf', 'sigmoid'], \n                     }\n\nSupport_Vector = SVM_Bagging_Randomize_CV(train_X,train_y, random_parameters, 15)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:09:50.403404Z","iopub.execute_input":"2022-06-30T14:09:50.403874Z","iopub.status.idle":"2022-06-30T14:23:07.111563Z","shell.execute_reply.started":"2022-06-30T14:09:50.403835Z","shell.execute_reply":"2022-06-30T14:23:07.110409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = Support_Vector.predict(test_X)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame({'prediction': Support_Vector.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['Support_Vector', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred), amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:23:07.113571Z","iopub.execute_input":"2022-06-30T14:23:07.114003Z","iopub.status.idle":"2022-06-30T14:23:56.810870Z","shell.execute_reply.started":"2022-06-30T14:23:07.113954Z","shell.execute_reply":"2022-06-30T14:23:56.809897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"def XGBoost_Randomize_CV(X,y, parameters, k = -1):\n    \n    # Create Parameter Combination\n    keys, values = zip(*parameters.items())\n    result = [dict(zip(keys, p)) for p in product(*values)]\n    \n    # If -1 Then Grid Search\n    if k != -1:\n\n        result = np.random.choice(result, k, replace = False)\n        \n    best_score = -1\n    best_parameter = {}\n    \n    # Train Test Split\n    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    for i in result:\n        \n\n        XGB_Model = xgb.XGBClassifier(**i,\n                                      early_stopping_rounds = 10,\n                                      verbosity = 0,\n                                       \n                                      n_jobs = -1).fit(train_X, train_y, eval_set = [(test_X, test_y)], verbose=False)\n        \n        pred = XGB_Model.predict(test_X).round().astype(int)\n        amex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\n        amex_prediction = pd.DataFrame({'prediction':XGB_Model.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\n        score = amex_metric(amex_actual,amex_prediction)\n        \n        if score > best_score:\n            best_score = score\n            best_parameter = i\n            best_model = XGB_Model\n        \n        print(f'{i}: {score}')\n\n        \n    print(f'Best Parameters - {best_parameter}: {best_score}')\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:23:56.812254Z","iopub.execute_input":"2022-06-30T14:23:56.814279Z","iopub.status.idle":"2022-06-30T14:23:56.825158Z","shell.execute_reply.started":"2022-06-30T14:23:56.814236Z","shell.execute_reply":"2022-06-30T14:23:56.823863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_parameters = {'max_depth':[9,10,11,12],\n                      'min_child_weight': [5,6,7,8],\n                      'eta':[.1, .01, .001],\n                      'objective':['binary:logistic'],\n                      'tree_method': ['gpu_hist'],\n                      'eval_metric': ['rmsle'],\n}\n\nXGB_Best = XGBoost_Randomize_CV(train_X,train_y, random_parameters)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:23:56.827000Z","iopub.execute_input":"2022-06-30T14:23:56.827617Z","iopub.status.idle":"2022-06-30T14:28:39.808226Z","shell.execute_reply.started":"2022-06-30T14:23:56.827573Z","shell.execute_reply":"2022-06-30T14:28:39.807261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = XGB_Best.predict(test_X).round().astype(int)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame({'prediction':XGB_Best.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['XGBC', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred), amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:28:39.809965Z","iopub.execute_input":"2022-06-30T14:28:39.810354Z","iopub.status.idle":"2022-06-30T14:28:43.366570Z","shell.execute_reply.started":"2022-06-30T14:28:39.810315Z","shell.execute_reply":"2022-06-30T14:28:43.365582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"def LGBM_Randomize_CV(X,y, parameters, k = -1):\n    \n    # Create Parameter Combination\n    keys, values = zip(*parameters.items())\n    result = [dict(zip(keys, p)) for p in product(*values)]\n    \n    # If -1 Then Grid Search\n    if k != -1:\n\n        result = np.random.choice(result, k, replace = False)\n        \n    best_score = -1\n    best_parameter = {}\n    best_model = None\n    \n    # Train Test Split\n    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    for i in result:\n        \n        LGBM_Model = lgbm.LGBMClassifier(**i,\n                                         device = 'gpu',\n                                         gpu_platform_id= 0,\n                                         gpu_device_id= 0,\n                                         n_jobs = -1).fit(train_X, train_y)\n        \n\n        pred = LGBM_Model.predict(test_X).round().astype(int)\n        amex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\n        amex_prediction = pd.DataFrame({'prediction':LGBM_Model.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\n        score = amex_metric(amex_actual,amex_prediction)\n        \n        if score > best_score:\n            best_score = score\n            best_parameter = i\n            best_model = LGBM_Model\n        \n        print(f'{i}: {score}')\n        \n    print(f'Best Parameters - {best_parameter}: {best_score}')\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:28:43.367957Z","iopub.execute_input":"2022-06-30T14:28:43.368572Z","iopub.status.idle":"2022-06-30T14:28:43.379960Z","shell.execute_reply.started":"2022-06-30T14:28:43.368514Z","shell.execute_reply":"2022-06-30T14:28:43.379029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_parameters = {'objective': ['binary'],\n                      'metric': ['binary_logloss'],\n                      'learning_rate':[0.1,0.01,0.001],\n                      'num_leaves':[6,7,8,9],\n                      'max_depth':[9,10,11,12]}\n\nLGBM_Model = LGBM_Randomize_CV(train_X,train_y, random_parameters, k = -1)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:28:43.381501Z","iopub.execute_input":"2022-06-30T14:28:43.381905Z","iopub.status.idle":"2022-06-30T14:31:01.236638Z","shell.execute_reply.started":"2022-06-30T14:28:43.381868Z","shell.execute_reply":"2022-06-30T14:31:01.235564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = LGBM_Model.predict(test_X)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame({'prediction': LGBM_Model.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['LGBM', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred), amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:31:01.238157Z","iopub.execute_input":"2022-06-30T14:31:01.238744Z","iopub.status.idle":"2022-06-30T14:31:03.689839Z","shell.execute_reply.started":"2022-06-30T14:31:01.238703Z","shell.execute_reply":"2022-06-30T14:31:03.688288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost","metadata":{}},{"cell_type":"code","source":"def CatBoost_Randomize_CV(X,y, parameters, k = -1):\n    \n    # Create Parameter Combination\n    keys, values = zip(*parameters.items())\n    result = [dict(zip(keys, p)) for p in product(*values)]\n    \n    # If -1 Then Grid Search\n    if k != -1:\n\n        result = np.random.choice(result, k, replace = False)\n        \n    best_score = -1\n    best_parameter = {}\n    best_model = None\n    \n    # Train Test Split\n    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    for i in result:\n        \n        CatBoost_Model = catboost.CatBoostClassifier(**i,\n                                                     verbose = 0,\n                                                     task_type=\"GPU\",\n                                                     devices='0',\n                                                     early_stopping_rounds = 10).fit(train_X, train_y)\n        \n\n        pred = CatBoost_Model.predict(test_X).round().astype(int)\n        amex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\n        amex_prediction = pd.DataFrame({'prediction':CatBoost_Model.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\n        score = amex_metric(amex_actual,amex_prediction)\n        \n        if score > best_score:\n            best_score = score\n            best_parameter = i\n            best_model = CatBoost_Model\n        \n        print(f'{i}: {score}')\n        \n    print(f'Best Parameters - {best_parameter}: {best_score}')\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:31:03.691257Z","iopub.execute_input":"2022-06-30T14:31:03.691750Z","iopub.status.idle":"2022-06-30T14:31:03.702772Z","shell.execute_reply.started":"2022-06-30T14:31:03.691710Z","shell.execute_reply":"2022-06-30T14:31:03.701789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_parameters = {'loss_function': ['Logloss'],\n                      'learning_rate':[0.1,0.01,0.001],\n                      'depth':[6,7,8,9,10]}\n\nCatBoost_Model = CatBoost_Randomize_CV(train_X,train_y, random_parameters, k = -1)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:31:03.704198Z","iopub.execute_input":"2022-06-30T14:31:03.704599Z","iopub.status.idle":"2022-06-30T14:36:37.281020Z","shell.execute_reply.started":"2022-06-30T14:31:03.704538Z","shell.execute_reply":"2022-06-30T14:36:37.279111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = CatBoost_Model.predict(test_X)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame({'prediction': CatBoost_Model.predict_proba(test_X)[:, 1]}).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['CatBoost', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred), amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:36:37.282911Z","iopub.execute_input":"2022-06-30T14:36:37.283329Z","iopub.status.idle":"2022-06-30T14:36:39.063121Z","shell.execute_reply.started":"2022-06-30T14:36:37.283284Z","shell.execute_reply":"2022-06-30T14:36:39.062206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:36:39.064696Z","iopub.execute_input":"2022-06-30T14:36:39.065308Z","iopub.status.idle":"2022-06-30T14:36:44.723696Z","shell.execute_reply.started":"2022-06-30T14:36:39.065266Z","shell.execute_reply":"2022-06-30T14:36:44.719974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.gpu_device_name()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:36:44.728450Z","iopub.execute_input":"2022-06-30T14:36:44.732834Z","iopub.status.idle":"2022-06-30T14:36:48.847909Z","shell.execute_reply.started":"2022-06-30T14:36:44.732790Z","shell.execute_reply":"2022-06-30T14:36:48.846438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining Early Stopping \nearly_stopping = EarlyStopping(\n    min_delta=0.001, # minimium amount of change to count as an improvement\n    patience=20, # how many epochs to wait before stopping\n    restore_best_weights=True,\n)\n\n# Defining Neural Model\nDL_Model =  keras.Sequential([\n    layers.Dense(32, input_shape = [6]),\n    layers.Dropout(.2,),\n    layers.Dense(16, activation = 'relu'),\n    layers.Dense(1, activation='sigmoid'),\n])\n\n# Compile Model Fit\nDL_Model.compile(\n    optimizer='adam',\n    loss='BinaryCrossentropy',\n    metrics = 'MeanSquaredLogarithmicError'\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:36:48.850167Z","iopub.execute_input":"2022-06-30T14:36:48.850917Z","iopub.status.idle":"2022-06-30T14:36:49.258390Z","shell.execute_reply.started":"2022-06-30T14:36:48.850875Z","shell.execute_reply":"2022-06-30T14:36:49.256789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Record The Epoch \n\nwith tf.device('/gpu:0'):\n    History = DL_Model.fit(train_X, train_y,\n                           validation_data=(test_X, test_y,),\n                           callbacks=[early_stopping],\n                           batch_size=500,\n                           epochs=1000,\n                           verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:36:49.259943Z","iopub.execute_input":"2022-06-30T14:36:49.260321Z","iopub.status.idle":"2022-06-30T14:38:31.115395Z","shell.execute_reply.started":"2022-06-30T14:36:49.260282Z","shell.execute_reply":"2022-06-30T14:38:31.114500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the Training History to a Dataframe\nhistory_df = pd.DataFrame(History.history)\n#Pandas native plot method\nhistory_df.loc[:, ['loss', 'val_loss']].plot();\nhistory_df.loc[:, ['mean_squared_logarithmic_error', 'val_mean_squared_logarithmic_error']].plot()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:38:31.116979Z","iopub.execute_input":"2022-06-30T14:38:31.117466Z","iopub.status.idle":"2022-06-30T14:39:12.527009Z","shell.execute_reply.started":"2022-06-30T14:38:31.117422Z","shell.execute_reply":"2022-06-30T14:39:12.526096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\npred = DL_Model.predict(test_X).round(0).astype(int)\namex_actual = pd.DataFrame({'target':test_y}).reset_index(drop=True)\namex_prediction = pd.DataFrame(DL_Model.predict(test_X), columns = ['prediction']).reset_index(drop=True)\nPerformance_df = Performance_df.append(pd.DataFrame([['Tensorflow', 'Full', accuracy_score(test_y, pred), log_loss(test_y, pred), roc_auc_score(test_y, pred),amex_metric(amex_actual,amex_prediction)]],\n                                                    columns = ['Model', 'Feature Selection','Accuracy', 'Log Loss', 'ROC','Amex Metric']), sort = False)\n\nprint('Accuracy:', accuracy_score(test_y, pred))\nprint('Log Loss:', log_loss(test_y, pred))\nprint('ROC Accuracy:', roc_auc_score(test_y, pred))\nprint('Confusion Matrix:\\n', \n       confusion_matrix(test_y, pred))\nprint('Amex Metric:\\n', amex_metric(amex_actual,amex_prediction))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:39:12.531317Z","iopub.execute_input":"2022-06-30T14:39:12.533702Z","iopub.status.idle":"2022-06-30T14:39:29.384125Z","shell.execute_reply.started":"2022-06-30T14:39:12.533660Z","shell.execute_reply":"2022-06-30T14:39:29.383240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"sns.set_theme()\nsns.scatterplot(data = Performance_df, x = 'Log Loss',y = 'Accuracy', hue = 'Model', legend = 'brief')","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:39:29.385412Z","iopub.execute_input":"2022-06-30T14:39:29.385846Z","iopub.status.idle":"2022-06-30T14:39:29.753364Z","shell.execute_reply.started":"2022-06-30T14:39:29.385807Z","shell.execute_reply":"2022-06-30T14:39:29.752427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\nsns.scatterplot(data = Performance_df, x = 'Amex Metric',y = 'Accuracy', hue = 'Model', legend = 'brief')","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:39:29.754624Z","iopub.execute_input":"2022-06-30T14:39:29.755100Z","iopub.status.idle":"2022-06-30T14:39:30.112623Z","shell.execute_reply.started":"2022-06-30T14:39:29.755051Z","shell.execute_reply":"2022-06-30T14:39:30.111794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(data = Performance_df, x = 'Model',y = 'Amex Metric', hue = 'Model')","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:39:30.114260Z","iopub.execute_input":"2022-06-30T14:39:30.114845Z","iopub.status.idle":"2022-06-30T14:39:30.566165Z","shell.execute_reply.started":"2022-06-30T14:39:30.114804Z","shell.execute_reply":"2022-06-30T14:39:30.565367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Performance_df","metadata":{"execution":{"iopub.status.busy":"2022-06-30T14:39:30.567587Z","iopub.execute_input":"2022-06-30T14:39:30.567974Z","iopub.status.idle":"2022-06-30T14:39:30.582407Z","shell.execute_reply.started":"2022-06-30T14:39:30.567936Z","shell.execute_reply":"2022-06-30T14:39:30.581200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the performance table, we found that XGBC model has the highest amex metric score. Let us use this model for our prediction submission. However, the testing data is enormous and we have memory limition. We will have part two to complete the submission!","metadata":{}}]}