{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Import packages\nimport numpy as np\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-17T11:16:50.462354Z","iopub.execute_input":"2022-08-17T11:16:50.462810Z","iopub.status.idle":"2022-08-17T11:16:50.491986Z","shell.execute_reply.started":"2022-08-17T11:16:50.462722Z","shell.execute_reply":"2022-08-17T11:16:50.490762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading the train data\ntrain_data = pd.read_parquet('../input/amex-parquet/train_data.parquet')\n                         \n#Explore train data\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:16:50.493571Z","iopub.execute_input":"2022-08-17T11:16:50.494555Z","iopub.status.idle":"2022-08-17T11:17:26.396856Z","shell.execute_reply.started":"2022-08-17T11:16:50.494488Z","shell.execute_reply":"2022-08-17T11:17:26.395750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check shape of train data\ntrain_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:26.398676Z","iopub.execute_input":"2022-08-17T11:17:26.399032Z","iopub.status.idle":"2022-08-17T11:17:26.405854Z","shell.execute_reply.started":"2022-08-17T11:17:26.399000Z","shell.execute_reply":"2022-08-17T11:17:26.404752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check for number of unique customers\nlen(train_data.customer_ID.unique())\n\n## We have 458913 unique customers.","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:26.409382Z","iopub.execute_input":"2022-08-17T11:17:26.410117Z","iopub.status.idle":"2022-08-17T11:17:27.325464Z","shell.execute_reply.started":"2022-08-17T11:17:26.410081Z","shell.execute_reply":"2022-08-17T11:17:27.324385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n# Check for number of missing values\ntrain_data.isnull().sum()\n\n## Could be observed that there are many columns with many missing values","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:27.327134Z","iopub.execute_input":"2022-08-17T11:17:27.327850Z","iopub.status.idle":"2022-08-17T11:17:30.129440Z","shell.execute_reply.started":"2022-08-17T11:17:27.327803Z","shell.execute_reply":"2022-08-17T11:17:30.128297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are multiple transactions. Lets take only the latest transaction from each customer.\ntrain=train_data.groupby('customer_ID').tail(1)\ntrain=train.set_index(['customer_ID'])\n\n#Drop date column since it is no longer relevant\ntrain.drop(['S_2'],axis=1,inplace=True)\n#Check for number of rows\ntrain.shape\n# We now have 458913 rows, which corresponds to the number of unique customers.","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:30.130795Z","iopub.execute_input":"2022-08-17T11:17:30.131231Z","iopub.status.idle":"2022-08-17T11:17:32.791529Z","shell.execute_reply.started":"2022-08-17T11:17:30.131201Z","shell.execute_reply":"2022-08-17T11:17:32.790192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Identify columns which are not numeric\ntrain.select_dtypes(['object'])\n\n##D_63 and D_64 turns out to be categorical but are strings","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:32.792899Z","iopub.execute_input":"2022-08-17T11:17:32.793240Z","iopub.status.idle":"2022-08-17T11:17:32.815730Z","shell.execute_reply.started":"2022-08-17T11:17:32.793210Z","shell.execute_reply":"2022-08-17T11:17:32.814669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Perform one-hot encoding for D_63 and D_64\n#Drop columns D_63 and D_64 subsequently\ntrain_D63 = pd.get_dummies(train[['D_63']])\ntrain = pd.concat([train, train_D63], axis=1)\ntrain = train.drop(['D_63'], axis=1)\n\ntrain_D64 = pd.get_dummies(train[['D_64']])\ntrain = pd.concat([train, train_D64], axis=1)\ntrain = train.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:32.817164Z","iopub.execute_input":"2022-08-17T11:17:32.817480Z","iopub.status.idle":"2022-08-17T11:17:34.156782Z","shell.execute_reply.started":"2022-08-17T11:17:32.817451Z","shell.execute_reply":"2022-08-17T11:17:34.155568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets check for the columns\ntrain.columns\n# We now have 196 columns including target\n# We need to reduce the dimensionality of the data","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:34.158260Z","iopub.execute_input":"2022-08-17T11:17:34.158649Z","iopub.status.idle":"2022-08-17T11:17:34.164997Z","shell.execute_reply.started":"2022-08-17T11:17:34.158614Z","shell.execute_reply":"2022-08-17T11:17:34.164081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Given that there are many columns with large number of missing values, it is impractical to go through every single one of them to determine whether it is useful. \n#Furthermore, we do not have information on the feature (e.g. actual name of the feature) except the type of variable\n#Lets remove columns if there are >85% of missing values\ntrain=train.dropna(axis=1, thresh=int(0.85*len(train)))\n\n#Checking the shape of new train data\ntrain.shape\n## We are left with 160 columns","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:34.168542Z","iopub.execute_input":"2022-08-17T11:17:34.168868Z","iopub.status.idle":"2022-08-17T11:17:34.457673Z","shell.execute_reply.started":"2022-08-17T11:17:34.168839Z","shell.execute_reply":"2022-08-17T11:17:34.456550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We shall remove highly correlated features.\ntrain_without_target=train.drop(['target'],axis=1)\ncor_matrix = train_without_target.corr().abs()\nupper_tri = cor_matrix.where((np.triu(np.ones(cor_matrix.shape), k=1) + np.tril(np.ones(cor_matrix.shape), k=-1)).astype(bool))\n#Drop out columns with absolute correlation of more than 85%\nto_drop = [column for column in upper_tri.columns if any(upper_tri[column] > 0.85)]\ntrain_drop_highcorr=train_without_target.drop(to_drop,axis=1)\ntrain_drop_highcorr.shape\n#We are now left with 131 columns (excluding target), which is still significant","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:17:34.459400Z","iopub.execute_input":"2022-08-17T11:17:34.459765Z","iopub.status.idle":"2022-08-17T11:18:06.260377Z","shell.execute_reply.started":"2022-08-17T11:17:34.459728Z","shell.execute_reply":"2022-08-17T11:18:06.259170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets remove columns with variance less than or equal to 0.05. Keep only columns with high variance.\nfrom sklearn.feature_selection import VarianceThreshold\nfrom itertools import compress\ndef fs_variance(df, threshold:float=0.05):\n    \"\"\"\n    Return a list of selected variables based on the threshold.\n    \"\"\"\n    # The list of columns in the data frame\n    features = list(df.columns)\n    \n    # Initialize and fit the method\n    vt = VarianceThreshold(threshold = threshold)\n    _ = vt.fit(df)\n    \n    # Get which column names which pass the threshold\n    feat_select = list(compress(features, vt.get_support()))\n    \n    return feat_select\ncolumns_to_keep=fs_variance(train_drop_highcorr)\n# We are left with 85 columns (excluding target), which passed the threshold.\ntrain_final=train[columns_to_keep]\nlen(columns_to_keep)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:18:06.263557Z","iopub.execute_input":"2022-08-17T11:18:06.264423Z","iopub.status.idle":"2022-08-17T11:18:07.797035Z","shell.execute_reply.started":"2022-08-17T11:18:06.264388Z","shell.execute_reply":"2022-08-17T11:18:07.795762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Concat target onto train_final\ntrain_final1=train_final.join(train['target'])\nx_train=train_final1.drop(['target'],axis=1)\ny_train=train_final1['target']","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:18:07.798623Z","iopub.execute_input":"2022-08-17T11:18:07.798992Z","iopub.status.idle":"2022-08-17T11:18:08.028116Z","shell.execute_reply.started":"2022-08-17T11:18:07.798960Z","shell.execute_reply":"2022-08-17T11:18:08.026896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split train data into training and testing sets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score  \nfrom sklearn.metrics import precision_score                         \nfrom sklearn.metrics import recall_score\nx_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train, test_size=0.25, random_state=26)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:18:08.029905Z","iopub.execute_input":"2022-08-17T11:18:08.030403Z","iopub.status.idle":"2022-08-17T11:18:08.412531Z","shell.execute_reply.started":"2022-08-17T11:18:08.030351Z","shell.execute_reply":"2022-08-17T11:18:08.411481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Comment out this section as it takes a very long time to run, the results of randomizedsearchCV is included below\n# Grid of hyperparameters to search over\n#from sklearn.model_selection import RandomizedSearchCV\n#param_random_gb = {'learning_rate': np.arange(0.05,0.55, 0.1),\n#                   'n_estimators' : [125,150,175],\n#                   'subsample' : np.arange(0.3,1.0, 0.1),\n#                   'max_depth':[3,4,5]}\n\n# Use XGBoost Classifier\nfrom xgboost import XGBClassifier\n\n# Perform RandomizedSearchCV\n#mse_random = RandomizedSearchCV(estimator = XGBClassifier(), param_distributions = param_random_gb, \n#                               n_iter = 10,scoring = 'neg_mean_squared_error', cv = 4, verbose = 1)\n\n#mse_random.fit(x_train_split,y_train_split)\n\n#print(\"Best parameter: \", mse_random.best_params_)\n#print(\"Lowest RMSE: \", np.sqrt(np.abs(mse_random.best_score_)))\n#Best parameter:  {'subsample': 0.5, 'n_estimators': 175, 'max_depth': 3, 'learning_rate': 0.15}\n#Lowest RMSE:  0.32263831733224874","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:18:08.414061Z","iopub.execute_input":"2022-08-17T11:18:08.414400Z","iopub.status.idle":"2022-08-17T11:18:08.540474Z","shell.execute_reply.started":"2022-08-17T11:18:08.414369Z","shell.execute_reply":"2022-08-17T11:18:08.539194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Run XGBoost model with the best parameters found\nmodel=XGBClassifier(n_estimators=200,max_depth=3,learning_rate=0.15, subsample=0.5)\nmodel.fit(x_train_split,y_train_split)\n#Test the model\ny_predict=model.predict(x_test_split)\nprint('XGBoost Classifier Accuracy: {:.3f}'.format(accuracy_score(y_test_split, y_predict)))\n# Achieved 89.5% accuracy","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:18:08.541741Z","iopub.execute_input":"2022-08-17T11:18:08.542095Z","iopub.status.idle":"2022-08-17T11:23:04.723954Z","shell.execute_reply.started":"2022-08-17T11:18:08.542062Z","shell.execute_reply":"2022-08-17T11:23:04.722590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nXGBoost Classifier Precision: {:.3f}'.format(precision_score (y_test_split, y_predict)))\n# Achieved Precision Score of 0.799","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:04.725876Z","iopub.execute_input":"2022-08-17T11:23:04.727739Z","iopub.status.idle":"2022-08-17T11:23:04.788837Z","shell.execute_reply.started":"2022-08-17T11:23:04.727689Z","shell.execute_reply":"2022-08-17T11:23:04.787260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nXGBoost Classifier Recall: {:.3f}'.format(recall_score (y_test_split, y_predict)))\n#Achieved Recall Score of 0.791","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:04.790594Z","iopub.execute_input":"2022-08-17T11:23:04.791079Z","iopub.status.idle":"2022-08-17T11:23:04.850539Z","shell.execute_reply.started":"2022-08-17T11:23:04.791046Z","shell.execute_reply":"2022-08-17T11:23:04.849323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of columns that we want to load for test data. Remove one-hot encoded names and target (since these columns not in the test data)\ncolumns_to_load=list(columns_to_keep)\ncolumns_to_load=columns_to_load+['D_63','D_64','customer_ID','S_2']\ncolumns_to_load.remove('D_63_CO')\ncolumns_to_load.remove('D_63_CR')\ncolumns_to_load.remove('D_63_CL')\ncolumns_to_load.remove('D_64_O')\ncolumns_to_load.remove('D_64_R')\ncolumns_to_load.remove('D_64_U')","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:04.852329Z","iopub.execute_input":"2022-08-17T11:23:04.852777Z","iopub.status.idle":"2022-08-17T11:23:04.859805Z","shell.execute_reply.started":"2022-08-17T11:23:04.852736Z","shell.execute_reply":"2022-08-17T11:23:04.858578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Read in the test_data\ntest_data = pd.read_parquet('../input/amex-parquet/test_data.parquet',columns=columns_to_load)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:04.861638Z","iopub.execute_input":"2022-08-17T11:23:04.862340Z","iopub.status.idle":"2022-08-17T11:23:43.247438Z","shell.execute_reply.started":"2022-08-17T11:23:04.862296Z","shell.execute_reply":"2022-08-17T11:23:43.246231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are multiple transactions. Lets take only the latest transaction from each customer.\ntest=test_data.groupby('customer_ID').tail(1)\ntest=test.set_index(['customer_ID'])\n\n#Drop date column since it is no longer relevant\ntest.drop(['S_2'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:43.248915Z","iopub.execute_input":"2022-08-17T11:23:43.249357Z","iopub.status.idle":"2022-08-17T11:23:45.289110Z","shell.execute_reply.started":"2022-08-17T11:23:43.249322Z","shell.execute_reply":"2022-08-17T11:23:45.287667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Perform one-hot encoding for D_63 and D_64\n#Drop columns D_63 and D_64 subsequently\ntest_D63 = pd.get_dummies(test[['D_63']])\ntest = pd.concat([test, test_D63], axis=1)\ntest = test.drop(['D_63'], axis=1)\n\ntest_D64 = pd.get_dummies(test[['D_64']])\ntest = pd.concat([test, test_D64], axis=1)\ntest = test.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:45.290640Z","iopub.execute_input":"2022-08-17T11:23:45.291013Z","iopub.status.idle":"2022-08-17T11:23:46.290210Z","shell.execute_reply.started":"2022-08-17T11:23:45.290979Z","shell.execute_reply":"2022-08-17T11:23:46.288950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Keep columns that we want.\ntest_final=test[columns_to_keep]","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:46.291672Z","iopub.execute_input":"2022-08-17T11:23:46.292032Z","iopub.status.idle":"2022-08-17T11:23:46.390185Z","shell.execute_reply.started":"2022-08-17T11:23:46.292000Z","shell.execute_reply":"2022-08-17T11:23:46.388983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict probabilities of default\ny_test_predict=model.predict_proba(test_final)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:46.391818Z","iopub.execute_input":"2022-08-17T11:23:46.392268Z","iopub.status.idle":"2022-08-17T11:23:48.134051Z","shell.execute_reply.started":"2022-08-17T11:23:46.392224Z","shell.execute_reply":"2022-08-17T11:23:48.132919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Retrieve the probability of default\ny_predict_final=y_test_predict[:,1]\n\n# Merge the prediction and customer_ID into submission dataframe\nsubmission = pd.DataFrame({\"customer_ID\":test_final.index,\"prediction\":y_predict_final})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T11:23:48.135524Z","iopub.execute_input":"2022-08-17T11:23:48.135875Z","iopub.status.idle":"2022-08-17T11:23:51.252522Z","shell.execute_reply.started":"2022-08-17T11:23:48.135843Z","shell.execute_reply":"2022-08-17T11:23:51.251355Z"},"trusted":true},"execution_count":null,"outputs":[]}]}