{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Import packages\nimport numpy as np\nimport pandas as pd\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T16:24:14.686389Z","iopub.execute_input":"2022-08-11T16:24:14.687233Z","iopub.status.idle":"2022-08-11T16:24:14.701646Z","shell.execute_reply.started":"2022-08-11T16:24:14.687148Z","shell.execute_reply":"2022-08-11T16:24:14.700275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading the train data\ntrain_data = pd.read_parquet('../input/amex-parquet/train_data.parquet')\n                         \n#Explore train data\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:24:14.710103Z","iopub.execute_input":"2022-08-11T16:24:14.710846Z","iopub.status.idle":"2022-08-11T16:24:57.007085Z","shell.execute_reply.started":"2022-08-11T16:24:14.710804Z","shell.execute_reply":"2022-08-11T16:24:57.005729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check shape of train data\ntrain_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:24:57.009249Z","iopub.execute_input":"2022-08-11T16:24:57.009783Z","iopub.status.idle":"2022-08-11T16:24:57.017333Z","shell.execute_reply.started":"2022-08-11T16:24:57.009730Z","shell.execute_reply":"2022-08-11T16:24:57.016318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check for number of unique customers\nlen(train_data.customer_ID.unique())\n\n## We have 458913 unique customers.","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:24:57.020504Z","iopub.execute_input":"2022-08-11T16:24:57.020936Z","iopub.status.idle":"2022-08-11T16:24:57.957029Z","shell.execute_reply.started":"2022-08-11T16:24:57.020896Z","shell.execute_reply":"2022-08-11T16:24:57.955663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n# Check for number of missing values\ntrain_data.isnull().sum()\n\n## Could be observed that there are many columns with many missing values","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:24:57.959138Z","iopub.execute_input":"2022-08-11T16:24:57.959723Z","iopub.status.idle":"2022-08-11T16:25:00.716768Z","shell.execute_reply.started":"2022-08-11T16:24:57.959655Z","shell.execute_reply":"2022-08-11T16:25:00.715412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets investigate Delinquency variables for missing values\nfilter_col_D=[col for col in train_data if col.startswith('D')]\ntrain_data[filter_col_D].isnull().sum()\n#There are many columns with large number of missing value","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:00.718695Z","iopub.execute_input":"2022-08-11T16:25:00.719487Z","iopub.status.idle":"2022-08-11T16:25:03.660036Z","shell.execute_reply.started":"2022-08-11T16:25:00.719434Z","shell.execute_reply":"2022-08-11T16:25:03.658661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets investigate Spend variables for missing values\nfilter_col_S=[col for col in train_data if col.startswith('S')]\ntrain_data[filter_col_S].isnull().sum()\n#There are many columns with large number of missing value","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:03.661715Z","iopub.execute_input":"2022-08-11T16:25:03.662218Z","iopub.status.idle":"2022-08-11T16:25:04.367893Z","shell.execute_reply.started":"2022-08-11T16:25:03.662156Z","shell.execute_reply":"2022-08-11T16:25:04.366298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets investigate Payment variables for missing values\nfilter_col_P=[col for col in train_data if col.startswith('P')]\ntrain_data[filter_col_P].isnull().sum()\n#There are 2 columns with large number of missing value","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:04.369485Z","iopub.execute_input":"2022-08-11T16:25:04.370346Z","iopub.status.idle":"2022-08-11T16:25:04.437446Z","shell.execute_reply.started":"2022-08-11T16:25:04.370264Z","shell.execute_reply":"2022-08-11T16:25:04.436083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets investigate Balance variables for missing values\nfilter_col_B=[col for col in train_data if col.startswith('B')]\ntrain_data[filter_col_B].isnull().sum()\n#There are many columns with large number of missing values","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:04.439464Z","iopub.execute_input":"2022-08-11T16:25:04.440385Z","iopub.status.idle":"2022-08-11T16:25:05.203894Z","shell.execute_reply.started":"2022-08-11T16:25:04.440329Z","shell.execute_reply":"2022-08-11T16:25:05.202532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets investigate Risk variables for missing values\nfilter_col_R=[col for col in train_data if col.startswith('R')]\ntrain_data[filter_col_R].isnull().sum()\n#There are 3 columns with large number of missing values","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:05.205865Z","iopub.execute_input":"2022-08-11T16:25:05.206358Z","iopub.status.idle":"2022-08-11T16:25:05.733327Z","shell.execute_reply.started":"2022-08-11T16:25:05.206307Z","shell.execute_reply":"2022-08-11T16:25:05.732313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for percentage of defaults to percentage of non-defaults\nPercentage=len(train_data[train_data['target']==1])*100/len(train_data[train_data['target']==0])\nPercentage\n#Only about 33.1% of the data are with defaults","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:05.734549Z","iopub.execute_input":"2022-08-11T16:25:05.735215Z","iopub.status.idle":"2022-08-11T16:25:10.993110Z","shell.execute_reply.started":"2022-08-11T16:25:05.735173Z","shell.execute_reply":"2022-08-11T16:25:10.991796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets remove columns if there are >90% of missing values.\n#Given that there are many columns with large number of missing values, it is impractical to go through every single one of them to determine whether it is useful. \n#Furthermore, we do not have information on the feature (e.g. actual name of the feature) except the type of variable\n#Brute force is thus a practical option to weed out columns with too many missing values.\n#Since about 33.1% of the data are defaults(66.9% non-defaults), it is safe to say that columns with >90% missing data are not useful.\ntrain=train_data.dropna(axis=1, thresh=int(0.90*len(train_data)))\n\n#Checking the shape of new train data\ntrain.shape\n## We are now left with 152 columns","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:10.994825Z","iopub.execute_input":"2022-08-11T16:25:10.995185Z","iopub.status.idle":"2022-08-11T16:25:15.656350Z","shell.execute_reply.started":"2022-08-11T16:25:10.995143Z","shell.execute_reply":"2022-08-11T16:25:15.655270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are multiple transactions. Lets take only the latest transaction from each customer.\n# Latest transaction may have missing values, we will perform forward fill for those missing values.\n# We perform forward fill as the last known value is likely to be brought forward to the next transaction.\n# We then do a backfill if the first row happens to be NA.\ntrain=train.set_index(['customer_ID'])\ntrain=train.ffill().bfill()\ntrain=train.reset_index()\ntrain=train.groupby('customer_ID').tail(1)\ntrain=train.set_index(['customer_ID'])\n\n#Drop date column since it is no longer relevant\ntrain.drop(['S_2'],axis=1,inplace=True)\n#Check for number of rows\ntrain.shape\n# We now have 458913 rows, which corresponds to the number of unique customers.","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:15.661724Z","iopub.execute_input":"2022-08-11T16:25:15.662173Z","iopub.status.idle":"2022-08-11T16:25:39.455469Z","shell.execute_reply.started":"2022-08-11T16:25:15.662130Z","shell.execute_reply":"2022-08-11T16:25:39.454055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets check again for missing values\ntrain.isnull().sum()\n# There are no more missing values","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:39.456851Z","iopub.execute_input":"2022-08-11T16:25:39.457206Z","iopub.status.idle":"2022-08-11T16:25:39.624157Z","shell.execute_reply.started":"2022-08-11T16:25:39.457173Z","shell.execute_reply":"2022-08-11T16:25:39.622961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Identify columns which are not numeric\ntrain.select_dtypes(['object'])\n\n##D_63 and D_64 turns out to be categorical but are strings","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:39.625815Z","iopub.execute_input":"2022-08-11T16:25:39.626321Z","iopub.status.idle":"2022-08-11T16:25:39.649916Z","shell.execute_reply.started":"2022-08-11T16:25:39.626268Z","shell.execute_reply":"2022-08-11T16:25:39.648725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Perform one-hot encoding for D_63 and D_64\n#Drop columns D_63 and D_64 subsequently\ntrain_D63 = pd.get_dummies(train[['D_63']])\ntrain = pd.concat([train, train_D63], axis=1)\ntrain = train.drop(['D_63'], axis=1)\n\ntrain_D64 = pd.get_dummies(train[['D_64']])\ntrain = pd.concat([train, train_D64], axis=1)\ntrain = train.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:39.651479Z","iopub.execute_input":"2022-08-11T16:25:39.652016Z","iopub.status.idle":"2022-08-11T16:25:40.856119Z","shell.execute_reply.started":"2022-08-11T16:25:39.651978Z","shell.execute_reply":"2022-08-11T16:25:40.854832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets check for the columns\ntrain.columns\n# We now have 158 columns including target\n# We need to reduce the dimensionality of the data","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:40.857537Z","iopub.execute_input":"2022-08-11T16:25:40.857937Z","iopub.status.idle":"2022-08-11T16:25:40.865553Z","shell.execute_reply.started":"2022-08-11T16:25:40.857901Z","shell.execute_reply":"2022-08-11T16:25:40.864323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:40.866992Z","iopub.execute_input":"2022-08-11T16:25:40.867999Z","iopub.status.idle":"2022-08-11T16:25:40.995772Z","shell.execute_reply.started":"2022-08-11T16:25:40.867951Z","shell.execute_reply":"2022-08-11T16:25:40.994881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We shall remove highly correlated features\ntrain_without_target=train.drop(['target'],axis=1)\ncor_matrix = train_without_target.corr().abs()\nupper_tri = cor_matrix.where(np.triu(np.ones(cor_matrix.shape),k=1).astype(np.bool))\n#Drop out columns with absolute correlation of more than 90%\nto_drop = [column for column in upper_tri.columns if any(upper_tri[column] > 0.90)]\ntrain_drop_highcorr=train.drop(to_drop,axis=1)\ntrain_drop_highcorr.shape\n#We are now left with 145 columns, which is still significant","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:25:40.996923Z","iopub.execute_input":"2022-08-11T16:25:40.997701Z","iopub.status.idle":"2022-08-11T16:26:12.244449Z","shell.execute_reply.started":"2022-08-11T16:25:40.997662Z","shell.execute_reply":"2022-08-11T16:26:12.242955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets remove columns with low variance=0.1. Keep only columns with high variance\nfrom sklearn.feature_selection import VarianceThreshold\nfrom itertools import compress\ndef fs_variance(df, threshold:float=0.1):\n    \"\"\"\n    Return a list of selected variables based on the threshold.\n    \"\"\"\n    # The list of columns in the data frame\n    features = list(df.columns)\n    \n    # Initialize and fit the method\n    vt = VarianceThreshold(threshold = threshold)\n    _ = vt.fit(df)\n    \n    # Get which column names which pass the threshold\n    feat_select = list(compress(features, vt.get_support()))\n    \n    return feat_select\ncolumns_to_keep=fs_variance(train_drop_highcorr)\n# We are left with 54 columns (excluding target), which passed the threshold.\ntrain_final=train[columns_to_keep]\nlen(columns_to_keep)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:12.246320Z","iopub.execute_input":"2022-08-11T16:26:12.246839Z","iopub.status.idle":"2022-08-11T16:26:14.532340Z","shell.execute_reply.started":"2022-08-11T16:26:12.246787Z","shell.execute_reply":"2022-08-11T16:26:14.530944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split the target into y. Remove target from x.\ny_train=train_final['target']\nx_train=train_final.drop(['target'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:14.533950Z","iopub.execute_input":"2022-08-11T16:26:14.535213Z","iopub.status.idle":"2022-08-11T16:26:14.580436Z","shell.execute_reply.started":"2022-08-11T16:26:14.535159Z","shell.execute_reply":"2022-08-11T16:26:14.579170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split train data into training and testing sets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score  \nfrom sklearn.metrics import precision_score                         \nfrom sklearn.metrics import recall_score\nx_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train, test_size=0.25, random_state=26)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:14.582599Z","iopub.execute_input":"2022-08-11T16:26:14.583419Z","iopub.status.idle":"2022-08-11T16:26:15.046030Z","shell.execute_reply.started":"2022-08-11T16:26:14.583370Z","shell.execute_reply":"2022-08-11T16:26:15.044606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Use Logistic regression\nfrom sklearn.linear_model import LogisticRegression \n#Train the model, with Lasso penalty term\nmodel=LogisticRegression(penalty='l2')\nmodel.fit(x_train_split,y_train_split)\n\n\ny_predict=model.predict(x_test_split)\nprint('\\nLogistics Regression Accuracy: {:.3f}'.format(accuracy_score(y_test_split, y_predict)))\n\n\nprint('\\nLogistics Regression Precision: {:.3f}'.format(precision_score (y_test_split, y_predict)))\n\n\nprint('\\nLogistics Regression Recall: {:.3f}'.format(recall_score (y_test_split, y_predict)))\n\n\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.047724Z","iopub.execute_input":"2022-08-11T16:26:15.048884Z","iopub.status.idle":"2022-08-11T16:26:15.058040Z","shell.execute_reply.started":"2022-08-11T16:26:15.048826Z","shell.execute_reply":"2022-08-11T16:26:15.056588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use Random Forest\nfrom sklearn.ensemble import RandomForestClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.059944Z","iopub.execute_input":"2022-08-11T16:26:15.060422Z","iopub.status.idle":"2022-08-11T16:26:15.176787Z","shell.execute_reply.started":"2022-08-11T16:26:15.060378Z","shell.execute_reply":"2022-08-11T16:26:15.175156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n# Number of trees in random forest\nn_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n# Number of features to consider at every split\nmax_features = ['auto', 'sqrt']\n# Maximum number of levels in tree\nmax_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\nmax_depth.append(None)\n# Minimum number of samples required to split a node\nmin_samples_split = [2, 5, 10]\n# Minimum number of samples required at each leaf node\nmin_samples_leaf = [1, 2, 4]\n# Method of selecting samples for training each tree\nbootstrap = [True, False]\n# Create the random grid\nrandom_grid = {'n_estimators': n_estimators,\n               'max_features': max_features,\n               'max_depth': max_depth,\n               'min_samples_split': min_samples_split,\n               'min_samples_leaf': min_samples_leaf,\n               'bootstrap': bootstrap}\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.178714Z","iopub.execute_input":"2022-08-11T16:26:15.179116Z","iopub.status.idle":"2022-08-11T16:26:15.188930Z","shell.execute_reply.started":"2022-08-11T16:26:15.179075Z","shell.execute_reply":"2022-08-11T16:26:15.186991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#rf = RandomForestClassifier()\n#rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 50, cv = 3, verbose=10, random_state=42, n_jobs = -1)\n#rf_random = GridSearchCV(estimator = rf, param_grid = random_grid, cv = 3, verbose=1, n_jobs = -1)\n# Fit the random search model\n#rf_random.fit(x_train_split,y_train_split)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.190033Z","iopub.execute_input":"2022-08-11T16:26:15.191148Z","iopub.status.idle":"2022-08-11T16:26:15.202412Z","shell.execute_reply.started":"2022-08-11T16:26:15.191108Z","shell.execute_reply":"2022-08-11T16:26:15.201053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rf_random.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.203629Z","iopub.execute_input":"2022-08-11T16:26:15.204055Z","iopub.status.idle":"2022-08-11T16:26:15.212467Z","shell.execute_reply.started":"2022-08-11T16:26:15.204016Z","shell.execute_reply":"2022-08-11T16:26:15.211396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=400, max_features='sqrt', bootstrap=True, max_depth=30, min_samples_leaf=1, min_samples_split=5, n_jobs=-1)\n#rf_random = GridSearchCV(estimator = rf, param_grid = random_grid, cv = 3, verbose=1, n_jobs = -1)\n# Fit the random search model\nmodel.fit(x_train_split,y_train_split)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:26:15.214503Z","iopub.execute_input":"2022-08-11T16:26:15.214994Z","iopub.status.idle":"2022-08-11T16:39:46.033900Z","shell.execute_reply.started":"2022-08-11T16:26:15.214950Z","shell.execute_reply":"2022-08-11T16:39:46.032448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef evaluate(model, test_features, test_labels):\n    predictions = model.predict(test_features)\n    errors = abs(predictions - test_labels)\n    mape = 100 * np.mean(errors / test_labels)\n    accuracy = 100 - mape\n    print('Model Performance')\n    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n    print('Accuracy = {:0.2f}%.'.format(accuracy))\n    \n    return accuracy\nbase_model = RandomForestClassifier(n_estimators = 10, random_state = 42)\nbase_model.fit(x_train_split, y_train_split)\nbase_accuracy = evaluate(base_model, x_test_split, y_test_split)\n\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:46.036049Z","iopub.execute_input":"2022-08-11T16:39:46.036839Z","iopub.status.idle":"2022-08-11T16:39:46.045279Z","shell.execute_reply.started":"2022-08-11T16:39:46.036796Z","shell.execute_reply":"2022-08-11T16:39:46.043601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#random_accuracy = evaluate(model, x_test_split, y_test_split)\n\n#print('Improvement of {:0.2f}%.'.format( 100 * (random_accuracy - base_accuracy) / base_accuracy))","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:46.047236Z","iopub.execute_input":"2022-08-11T16:39:46.047868Z","iopub.status.idle":"2022-08-11T16:39:46.056705Z","shell.execute_reply.started":"2022-08-11T16:39:46.047829Z","shell.execute_reply":"2022-08-11T16:39:46.055535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train_split, y_train_split\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:46.058413Z","iopub.execute_input":"2022-08-11T16:39:46.058799Z","iopub.status.idle":"2022-08-11T16:39:46.235473Z","shell.execute_reply.started":"2022-08-11T16:39:46.058763Z","shell.execute_reply":"2022-08-11T16:39:46.234226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of columns that we want to load for test data. Remove one-hot encoded names and target (since these columns not in the test data)\ncolumns_to_keep1=list(columns_to_keep)\ncolumns_to_keep1=columns_to_keep1+['D_63','D_64','customer_ID']\ncolumns_to_keep1.remove('D_63_CO')\ncolumns_to_keep1.remove('D_63_CR')\ncolumns_to_keep1.remove('D_64_O')\ncolumns_to_keep1.remove('D_64_R')\ncolumns_to_keep1.remove('D_64_U')\ncolumns_to_keep1.remove('target')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:46.237012Z","iopub.execute_input":"2022-08-11T16:39:46.237351Z","iopub.status.idle":"2022-08-11T16:39:46.243283Z","shell.execute_reply.started":"2022-08-11T16:39:46.237317Z","shell.execute_reply":"2022-08-11T16:39:46.242034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Read in the test_data\ntest_data = pd.read_parquet('../input/amex-parquet/test_data.parquet',columns=columns_to_keep1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:46.245435Z","iopub.execute_input":"2022-08-11T16:39:46.245831Z","iopub.status.idle":"2022-08-11T16:40:11.697550Z","shell.execute_reply.started":"2022-08-11T16:39:46.245794Z","shell.execute_reply":"2022-08-11T16:40:11.696472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are multiple transactions. Lets take only the latest transaction from each customer.\n# Latest transaction may have missing values, we will perform forward fill for those missing values.\n# We do a backfill if the first row happens to be Na\ntest=test_data.set_index(['customer_ID'])\ntest=test.ffill().bfill()\ntest=test.reset_index()\ntest=test.groupby('customer_ID').tail(1)\ntest=test.set_index(['customer_ID'])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:40:11.699048Z","iopub.execute_input":"2022-08-11T16:40:11.700285Z","iopub.status.idle":"2022-08-11T16:40:24.151298Z","shell.execute_reply.started":"2022-08-11T16:40:11.700241Z","shell.execute_reply":"2022-08-11T16:40:24.150038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Perform one-hot encoding for D_63 and D_64\n#Drop columns D_63 and D_64 subsequently\ntest_D63 = pd.get_dummies(test[['D_63']])\ntest = pd.concat([test, test_D63], axis=1)\ntest = test.drop(['D_63'], axis=1)\n\ntest_D64 = pd.get_dummies(test[['D_64']])\ntest = pd.concat([test, test_D64], axis=1)\ntest = test.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:40:24.152714Z","iopub.execute_input":"2022-08-11T16:40:24.153076Z","iopub.status.idle":"2022-08-11T16:40:25.076588Z","shell.execute_reply.started":"2022-08-11T16:40:24.153041Z","shell.execute_reply":"2022-08-11T16:40:25.075110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Keep columns that we want.\ncolumns_to_keep.remove('target')\ntest_final=test[columns_to_keep]","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:40:25.078113Z","iopub.execute_input":"2022-08-11T16:40:25.078492Z","iopub.status.idle":"2022-08-11T16:40:25.163887Z","shell.execute_reply.started":"2022-08-11T16:40:25.078457Z","shell.execute_reply":"2022-08-11T16:40:25.162140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict probabilities of default\ny_test_predict=model.predict_proba(test_final)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:40:25.165871Z","iopub.execute_input":"2022-08-11T16:40:25.166724Z","iopub.status.idle":"2022-08-11T16:41:15.133838Z","shell.execute_reply.started":"2022-08-11T16:40:25.166671Z","shell.execute_reply":"2022-08-11T16:41:15.132300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Retrieve the probability of default\ny_predict_final=y_test_predict[:,1]\n\n#Reset index of test\ntest=test.reset_index()\n\n# Merge the prediction and customer_ID into submission dataframe\nsubmission = pd.DataFrame({\"customer_ID\":test.customer_ID,\"prediction\":y_predict_final})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:52:07.027712Z","iopub.execute_input":"2022-08-11T16:52:07.028240Z","iopub.status.idle":"2022-08-11T16:52:11.342863Z","shell.execute_reply.started":"2022-08-11T16:52:07.028202Z","shell.execute_reply":"2022-08-11T16:52:11.341462Z"},"trusted":true},"execution_count":null,"outputs":[]}]}