{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"As noted in this [discussion](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327926), Private LB and Public LB seem to be split chronologically.  \nWe believe that analysis of these data will be important to avoid Shake down.  \nIn this Notebook, we would like to try Adversarial Validation as one of the methods.  \nThis result is based on the analysis of Private and Public LBs, and it does not mean that Shake down will occur.","metadata":{}},{"cell_type":"code","source":"import cudf\nimport cupy\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom catboost import CatBoost\nfrom catboost import Pool\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-08T22:32:31.681133Z","iopub.execute_input":"2022-06-08T22:32:31.681754Z","iopub.status.idle":"2022-06-08T22:32:31.692688Z","shell.execute_reply.started":"2022-06-08T22:32:31.681705Z","shell.execute_reply":"2022-06-08T22:32:31.691645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:26:15.109229Z","iopub.execute_input":"2022-06-08T22:26:15.109800Z","iopub.status.idle":"2022-06-08T22:26:50.868559Z","shell.execute_reply.started":"2022-06-08T22:26:15.109752Z","shell.execute_reply":"2022-06-08T22:26:50.867647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.drop_duplicates(subset=[\"customer_ID\"], keep=\"last\")\ntest['S_2'] = cudf.to_datetime(test['S_2'])\ntest['month'] = (test['S_2'].dt.month).astype('int8')\ntest = test.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:31:54.533250Z","iopub.execute_input":"2022-06-08T22:31:54.533987Z","iopub.status.idle":"2022-06-08T22:31:54.582063Z","shell.execute_reply.started":"2022-06-08T22:31:54.533949Z","shell.execute_reply":"2022-06-08T22:31:54.581176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['month'].value_counts(normalize = True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:26:54.298188Z","iopub.execute_input":"2022-06-08T22:26:54.302962Z","iopub.status.idle":"2022-06-08T22:26:54.362834Z","shell.execute_reply.started":"2022-06-08T22:26:54.302915Z","shell.execute_reply":"2022-06-08T22:26:54.361825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['private'] = 0\ntest.loc[test['month'] == 4,'private'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:26:54.368383Z","iopub.execute_input":"2022-06-08T22:26:54.371350Z","iopub.status.idle":"2022-06-08T22:26:56.212946Z","shell.execute_reply.started":"2022-06-08T22:26:54.371292Z","shell.execute_reply":"2022-06-08T22:26:56.211451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_params = {\n        'loss_function' : 'Logloss',\n        'eval_metric' : 'AUC',\n        'learning_rate': 0.08,\n        'num_boost_round': 5000,\n        'early_stopping_rounds': 100,\n        'random_state': 127,\n        'task_type': 'GPU'\n    }","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:31:59.435464Z","iopub.execute_input":"2022-06-08T22:31:59.436314Z","iopub.status.idle":"2022-06-08T22:31:59.441070Z","shell.execute_reply.started":"2022-06-08T22:31:59.436275Z","shell.execute_reply":"2022-06-08T22:31:59.440001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = KFold(n_splits=3)\ntest['fold'] = 999\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(test)):\n    test.loc[test.index.isin(idx_va),'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:32:00.273282Z","iopub.execute_input":"2022-06-08T22:32:00.273692Z","iopub.status.idle":"2022-06-08T22:32:00.388267Z","shell.execute_reply.started":"2022-06-08T22:32:00.273658Z","shell.execute_reply":"2022-06-08T22:32:00.387305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET = 'private'\ndrop_cols = ['S_2','month','customer_ID','fold',TARGET]\nuse_cols = [c for c in test.columns if c not in drop_cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:32:14.783783Z","iopub.execute_input":"2022-06-08T22:32:14.784948Z","iopub.status.idle":"2022-06-08T22:32:14.795228Z","shell.execute_reply.started":"2022-06-08T22:32:14.784902Z","shell.execute_reply":"2022-06-08T22:32:14.794348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = cupy.zeros(len(test))\nfi_df = pd.DataFrame()\nfor fold in range(3):\n    train_x,train_y = test[test['fold'] != fold][use_cols],test[test['fold'] != fold][TARGET]\n    valid_x,valid_y = test[test['fold'] == fold][use_cols],test[test['fold'] == fold][TARGET]\n\n    trn_data = Pool(train_x.to_pandas(), label=train_y.to_array())\n    val_data = Pool(valid_x.to_pandas(), label=valid_y.to_array())\n\n    model = CatBoost(cat_params)\n    model.fit(trn_data,\n            eval_set=val_data,\n            verbose_eval=500,\n            use_best_model=True\n          )\n\n\n    pred = model.predict(val_data)\n    auc_score = roc_auc_score(valid_y.to_array(),pred)\n    oof[valid_x.index] = pred\n\n    fi_df[f'fold_{fold}'] = model.get_feature_importance(Pool(train_x.to_pandas(), train_y.to_array()))","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:32:34.047319Z","iopub.execute_input":"2022-06-08T22:32:34.048255Z","iopub.status.idle":"2022-06-08T22:38:50.336402Z","shell.execute_reply.started":"2022-06-08T22:32:34.048213Z","shell.execute_reply":"2022-06-08T22:38:50.335441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi_df['importance'] = fi_df.mean(axis=1)\nfi_df['features'] = use_cols\nplt.figure(figsize=(10, 10))\nsns.barplot(x=\"importance\", y=\"features\", data=fi_df.sort_values(by=\"importance\", ascending=False)[:30])\nplt.title('CatBoost Features')\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:38:50.338038Z","iopub.execute_input":"2022-06-08T22:38:50.338544Z","iopub.status.idle":"2022-06-08T22:38:50.868088Z","shell.execute_reply.started":"2022-06-08T22:38:50.338503Z","shell.execute_reply":"2022-06-08T22:38:50.867192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The AUC score is above 0.99 and a check of the feature importance shows a significant difference in 'B_29'.","metadata":{}},{"cell_type":"code","source":"test = test.to_pandas()\nplt.hist(test[(test['private'] == 0) & (test['B_29']<0.02)]['B_29'],label='public')\nplt.hist(test[(test['private'] == 1) & (test['B_29']<0.02)]['B_29'],label='private')\nplt.legend()\nplt.xlim(0,0.03)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:39:17.155118Z","iopub.execute_input":"2022-06-08T22:39:17.155500Z","iopub.status.idle":"2022-06-08T22:39:18.994416Z","shell.execute_reply.started":"2022-06-08T22:39:17.155469Z","shell.execute_reply":"2022-06-08T22:39:18.993487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('public B_29:',test[test['private'] == 0]['B_29'].isnull().sum())\nprint('private B_29:',test[test['private'] == 1]['B_29'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-06-08T22:39:29.396785Z","iopub.execute_input":"2022-06-08T22:39:29.397172Z","iopub.status.idle":"2022-06-08T22:39:29.803617Z","shell.execute_reply.started":"2022-06-08T22:39:29.397142Z","shell.execute_reply":"2022-06-08T22:39:29.802649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above shows that 'B_29' needs to be analyzed in depth.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}