{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-04T18:55:39.507171Z","iopub.execute_input":"2023-04-04T18:55:39.507912Z","iopub.status.idle":"2023-04-04T18:55:39.516889Z","shell.execute_reply.started":"2023-04-04T18:55:39.507872Z","shell.execute_reply":"2023-04-04T18:55:39.515575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\n\n# Ignore all warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_rows', None)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:26:26.025554Z","iopub.execute_input":"2023-04-04T19:26:26.026518Z","iopub.status.idle":"2023-04-04T19:26:26.030962Z","shell.execute_reply.started":"2023-04-04T19:26:26.026480Z","shell.execute_reply":"2023-04-04T19:26:26.029633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nchunk = 100000\niterator = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=chunk)\ntrain_labels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\n\ntrain_data=[]\ntest1_data=[]\ntest2_data=[]\n\nfor ch in iterator:\n    ch = ch.sample(frac=1)\n    one_hot_df = pd.get_dummies(ch[['D_63','D_64']])\n    chunk = pd.concat([ch,one_hot_df], axis = 1)\n    chunk.drop(columns = ['D_63','D_64'], axis = 1, inplace = True)\n    \n    chunk = chunk.rename(columns={'S_2': 'Date'})\n    \n    chunk['Date'] = pd.to_datetime(chunk['Date'])\n    # Split the chunk into train and test sets\n\n    train_chunk = chunk[(chunk['Date'] >= '2017-05-01') & (chunk['Date'] <= '2018-01-31')]\n    test1_chunk = chunk[(chunk['Date'] >= '2017-03-01') & (chunk['Date'] <= '2017-04-30')]\n    test2_chunk = chunk[(chunk['Date'] >= '2018-02-01') & (chunk['Date'] <= '2018-03-31')]\n    \n    # Drop Date columns\n    train_chunk.drop(['Date'], axis=1, inplace = True)\n    test1_chunk.drop(['Date'], axis=1, inplace = True)\n    test2_chunk.drop(['Date'], axis=1, inplace = True)\n\n    \n    fftrain_chunk = train_chunk.groupby('customer_ID',as_index=False).median()\n    fftest1_chunk = test1_chunk.groupby('customer_ID',as_index=False).median()\n    fftest2_chunk = test2_chunk.groupby('customer_ID',as_index=False).median()\n    \n    train_data.append(fftrain_chunk)\n    test1_data.append(fftest1_chunk)\n    test2_data.append(fftest2_chunk)\n    \n    \n# Concatenate the train and test data\ntrain_data = pd.concat(train_data)\ntest1_data = pd.concat(test1_data)\ntest2_data = pd.concat(test2_data)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T18:55:43.812625Z","iopub.execute_input":"2023-04-04T18:55:43.813305Z","iopub.status.idle":"2023-04-04T19:03:01.743814Z","shell.execute_reply.started":"2023-04-04T18:55:43.813266Z","shell.execute_reply":"2023-04-04T19:03:01.742635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loading train labels and joining the two datasets with customer_ID\ndf_labels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\ndf_train_combined = pd.merge(train_data,df_labels, on='customer_ID')\ndf_test1_combined = pd.merge(test1_data,df_labels, on='customer_ID')\ndf_test2_combined = pd.merge(test2_data,df_labels, on='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:03:01.746070Z","iopub.execute_input":"2023-04-04T19:03:01.746442Z","iopub.status.idle":"2023-04-04T19:03:06.131818Z","shell.execute_reply.started":"2023-04-04T19:03:01.746402Z","shell.execute_reply":"2023-04-04T19:03:06.130733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_combined.drop(columns = 'customer_ID', axis = 1, inplace = True)\ndf_test1_combined.drop(columns = 'customer_ID', axis = 1, inplace = True)\ndf_test2_combined.drop(columns = 'customer_ID', axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:03:06.133495Z","iopub.execute_input":"2023-04-04T19:03:06.133872Z","iopub.status.idle":"2023-04-04T19:03:08.190131Z","shell.execute_reply.started":"2023-04-04T19:03:06.133834Z","shell.execute_reply":"2023-04-04T19:03:08.188985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = df_train_combined.drop(columns = ['target'], axis = 1)\ny_train = df_train_combined['target']\n\nX_test1 = df_test1_combined.drop(columns = ['target'], axis = 1)\ny_test1 = df_test1_combined['target']\n\nX_test2 = df_test2_combined.drop(columns = ['target'], axis = 1)\ny_test2 = df_test2_combined['target']","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:03:08.193341Z","iopub.execute_input":"2023-04-04T19:03:08.194107Z","iopub.status.idle":"2023-04-04T19:03:10.268467Z","shell.execute_reply.started":"2023-04-04T19:03:08.194067Z","shell.execute_reply":"2023-04-04T19:03:10.267200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Feature importance\nimport xgboost as xgb\n\nxgb_instance = xgb.XGBClassifier(tree_method='gpu_hist')\nmodel_for_feature_selection = xgb_instance.fit(X_train, y_train)\n\nfeature_importance = {'Feature':X_train.columns,'Importance':model_for_feature_selection.feature_importances_}\nfeature_importance = pd.DataFrame(feature_importance)\nfeature_importance.sort_values(\"Importance\", inplace=True,ascending=False)\nfeature_importance.to_csv('feature_importance_model1.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance.to_csv('feature_importance.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:03:41.404511Z","iopub.execute_input":"2023-04-04T19:03:41.404884Z","iopub.status.idle":"2023-04-04T19:03:42.226095Z","shell.execute_reply.started":"2023-04-04T19:03:41.404851Z","shell.execute_reply":"2023-04-04T19:03:42.225094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# XGB Params as per step 8\nparams = {\n    'learning_rate': 0.5,\n    'max_depth': 4,\n    'subsample': 0.5,\n    'colsample_bytree': 0.5,\n    'scale_pos_weight': 5,\n    'objective': 'binary:logistic',\n    'n_estimators': 300,\n    'random_state': 42\n}\n\n\n# XGBoost model\nxgb_model = xgb.XGBClassifier(**params, tree_method = 'gpu_hist')\nxgb_model.fit(X_train, y_train)\n\n# Get the feature importance\nimportance = xgb_model.feature_importances_\n\n# Create a DataFrame of feature names and their importance\nfeature_importance_model2 = pd.DataFrame({'Feature': X_train.columns, 'Importance': importance})\n\n# Sort the DataFrame in descending order of feature importance\nfeature_importance_model2 = feature_importance_model2.sort_values(by='Importance', ascending=False)\n\n# Save the feature importance as a CSV file\nfeature_importance_model2.to_csv('feature_importance_model2.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:04:01.956224Z","iopub.execute_input":"2023-04-04T19:04:01.956595Z","iopub.status.idle":"2023-04-04T19:04:15.415471Z","shell.execute_reply.started":"2023-04-04T19:04:01.956560Z","shell.execute_reply":"2023-04-04T19:04:15.414099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance_model2.sort_values(by='Importance', ascending = False)","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:04:25.443456Z","iopub.execute_input":"2023-04-04T19:04:25.444562Z","iopub.status.idle":"2023-04-04T19:04:25.465051Z","shell.execute_reply.started":"2023-04-04T19:04:25.444517Z","shell.execute_reply":"2023-04-04T19:04:25.463936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_features = feature_importance_model2[\"Feature\"][feature_importance_model2.Importance > 0.005]\n\nX_train_model2 = X_train[final_features]\nX_test1_model2 = X_test1[final_features]\nX_test2_model2 = X_test2[final_features]","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:04:32.748724Z","iopub.execute_input":"2023-04-04T19:04:32.749380Z","iopub.status.idle":"2023-04-04T19:04:32.859497Z","shell.execute_reply.started":"2023-04-04T19:04:32.749342Z","shell.execute_reply":"2023-04-04T19:04:32.858218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_features.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:04:34.132860Z","iopub.execute_input":"2023-04-04T19:04:34.133409Z","iopub.status.idle":"2023-04-04T19:04:34.140224Z","shell.execute_reply.started":"2023-04-04T19:04:34.133372Z","shell.execute_reply":"2023-04-04T19:04:34.139173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.metrics import roc_auc_score\ntable = pd.DataFrame(columns = [\"Num Trees\", \"Learning Rate\", \"Subsample\",\"pct_features\",\"wt_of_default\",\"AUC Train\", \"AUC Test1\", \"AUC Test2\"])\n\nrow = 0\nfor num_trees in [50, 100, 300]:\n    for LR in [0.01, 0.1]:\n        for subsample in [0.5, 0.8]:\n            for pctfeatures in [0.5,1]:\n                for wtd in [1,5,10]:\n                    xgb_instance = xgb.XGBClassifier(tree_method = 'gpu_hist',n_estimators=num_trees, learning_rate = LR, subsample= subsample, colsample_bytree= pctfeatures,scale_pos_weight = wtd, random_state = 42)\n                    model = xgb_instance.fit(X_train_model2, y_train)\n\n                    table.loc[row,\"Num Trees\"] = num_trees\n                    table.loc[row,\"Learning Rate\"] = LR\n                    table.loc[row,\"Subsample\"] = subsample\n                    table.loc[row,\"pct_features\"] = pctfeatures\n                    table.loc[row,\"wt_of_default\"] = wtd\n                    table.loc[row,\"AUC Train\"] = roc_auc_score(y_train, model.predict_proba(X_train_model2)[:,1])\n                    table.loc[row,\"AUC Test1\"] = roc_auc_score(y_test1, model.predict_proba(X_test1_model2)[:,1])\n                    table.loc[row,\"AUC Test2\"] = roc_auc_score(y_test2, model.predict_proba(X_test2_model2)[:,1])\n\n                    row = row + 1\n\ntable","metadata":{"execution":{"iopub.status.busy":"2023-04-04T19:04:50.668820Z","iopub.execute_input":"2023-04-04T19:04:50.669898Z","iopub.status.idle":"2023-04-04T19:14:30.028387Z","shell.execute_reply.started":"2023-04-04T19:04:50.669831Z","shell.execute_reply":"2023-04-04T19:14:30.027338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table.to_csv('xgboost_grid.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]}]}