{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cudf\nimport cupy\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom catboost import CatBoost\nfrom catboost import Pool\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom catboost import CatBoostClassifier\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-11T10:20:03.704097Z","iopub.execute_input":"2022-06-11T10:20:03.704694Z","iopub.status.idle":"2022-06-11T10:20:08.754791Z","shell.execute_reply.started":"2022-06-11T10:20:03.704546Z","shell.execute_reply":"2022-06-11T10:20:08.753324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_adversarial_data(df_train, df_test, cols, N_val=70000):\n    df_master = df_train[cols].append(df_test[cols], ignore_index=True)\n    adversarial_val = df_master.sample(N_val, replace=False)\n    adversarial_train = df_master[\n        ~df_master.index.isin(adversarial_val.index)\n    ]\n    return adversarial_train, adversarial_val","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:20:08.757195Z","iopub.execute_input":"2022-06-11T10:20:08.757615Z","iopub.status.idle":"2022-06-11T10:20:08.773124Z","shell.execute_reply.started":"2022-06-11T10:20:08.757569Z","shell.execute_reply":"2022-06-11T10:20:08.771494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This work inspired me to create this notebook - https://www.kaggle.com/code/zakopur0/adversarial-validation-private-vs-public/notebook","metadata":{}},{"cell_type":"markdown","source":"## This notebook is an extension of the notebook: https://www.kaggle.com/code/mikhaildonskoy/looking-for-risky-features-in-train-data","metadata":{}},{"cell_type":"markdown","source":"## In this work, I want to find out which features differ the most on the test and training dataset, which can lead to overfitting \n## Unlike previous work, I will take the first lines for each customer","metadata":{}},{"cell_type":"markdown","source":"# Data preparation","metadata":{}},{"cell_type":"code","source":"train = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet').drop_duplicates(subset=[\"customer_ID\"], keep=\"first\")\ntest = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet').drop_duplicates(subset=[\"customer_ID\"], keep=\"first\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:21:44.249808Z","iopub.execute_input":"2022-06-11T10:21:44.250204Z","iopub.status.idle":"2022-06-11T10:21:53.518662Z","shell.execute_reply.started":"2022-06-11T10:21:44.250173Z","shell.execute_reply":"2022-06-11T10:21:53.517589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"is_train\"] = 1\ntest[\"is_train\"] = 0\ntarget = ['is_train']\ndrop_cols = ['S_2','customer_ID',\"is_train\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:22:00.328711Z","iopub.execute_input":"2022-06-11T10:22:00.329148Z","iopub.status.idle":"2022-06-11T10:22:02.626082Z","shell.execute_reply.started":"2022-06-11T10:22:00.329116Z","shell.execute_reply":"2022-06-11T10:22:02.624942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Starting Negative Feature Selection","metadata":{}},{"cell_type":"code","source":"train_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:22:04.350922Z","iopub.execute_input":"2022-06-11T10:22:04.351355Z","iopub.status.idle":"2022-06-11T10:22:06.199070Z","shell.execute_reply.started":"2022-06-11T10:22:04.351324Z","shell.execute_reply":"2022-06-11T10:22:06.198073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:22:07.897772Z","iopub.execute_input":"2022-06-11T10:22:07.898247Z","iopub.status.idle":"2022-06-11T10:23:56.599515Z","shell.execute_reply.started":"2022-06-11T10:22:07.898217Z","shell.execute_reply":"2022-06-11T10:23:56.598420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:23:59.534842Z","iopub.execute_input":"2022-06-11T10:23:59.535482Z","iopub.status.idle":"2022-06-11T10:24:00.247756Z","shell.execute_reply.started":"2022-06-11T10:23:59.535433Z","shell.execute_reply":"2022-06-11T10:24:00.246570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop R1","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:24:07.235330Z","iopub.execute_input":"2022-06-11T10:24:07.236354Z","iopub.status.idle":"2022-06-11T10:24:21.078573Z","shell.execute_reply.started":"2022-06-11T10:24:07.236315Z","shell.execute_reply":"2022-06-11T10:24:21.077612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S_11","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"S_11\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:25:03.128889Z","iopub.execute_input":"2022-06-11T10:25:03.129339Z","iopub.status.idle":"2022-06-11T10:25:14.747492Z","shell.execute_reply.started":"2022-06-11T10:25:03.129308Z","shell.execute_reply":"2022-06-11T10:25:14.746615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_59","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"S_11\",\"D_59\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:27:44.024938Z","iopub.execute_input":"2022-06-11T10:27:44.025369Z","iopub.status.idle":"2022-06-11T10:27:54.693473Z","shell.execute_reply.started":"2022-06-11T10:27:44.025338Z","shell.execute_reply":"2022-06-11T10:27:54.692598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_121","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:41:44.163554Z","iopub.execute_input":"2022-06-11T10:41:44.164003Z","iopub.status.idle":"2022-06-11T10:41:55.072550Z","shell.execute_reply.started":"2022-06-11T10:41:44.163956Z","shell.execute_reply":"2022-06-11T10:41:55.071651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S_27","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:42:24.349174Z","iopub.execute_input":"2022-06-11T10:42:24.349657Z","iopub.status.idle":"2022-06-11T10:42:34.676164Z","shell.execute_reply.started":"2022-06-11T10:42:24.349627Z","shell.execute_reply":"2022-06-11T10:42:34.675173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_118","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:43:50.873450Z","iopub.execute_input":"2022-06-11T10:43:50.873909Z","iopub.status.idle":"2022-06-11T10:44:02.326413Z","shell.execute_reply.started":"2022-06-11T10:43:50.873879Z","shell.execute_reply":"2022-06-11T10:44:02.325559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_119","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:44:38.757383Z","iopub.execute_input":"2022-06-11T10:44:38.757796Z","iopub.status.idle":"2022-06-11T10:44:49.191636Z","shell.execute_reply.started":"2022-06-11T10:44:38.757753Z","shell.execute_reply":"2022-06-11T10:44:49.190794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_120","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:46:07.207369Z","iopub.execute_input":"2022-06-11T10:46:07.207797Z","iopub.status.idle":"2022-06-11T10:46:18.114753Z","shell.execute_reply.started":"2022-06-11T10:46:07.207754Z","shell.execute_reply":"2022-06-11T10:46:18.113847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop P_4","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\",\"P_4\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:46:47.424028Z","iopub.execute_input":"2022-06-11T10:46:47.424641Z","iopub.status.idle":"2022-06-11T10:46:57.581712Z","shell.execute_reply.started":"2022-06-11T10:46:47.424597Z","shell.execute_reply":"2022-06-11T10:46:57.580791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_39","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\",\"P_4\",\"D_39\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:48:18.636120Z","iopub.execute_input":"2022-06-11T10:48:18.636766Z","iopub.status.idle":"2022-06-11T10:48:29.651870Z","shell.execute_reply.started":"2022-06-11T10:48:18.636692Z","shell.execute_reply":"2022-06-11T10:48:29.650964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop R_27","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\",\"P_4\",\"D_39\",\"R_27\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:49:16.455713Z","iopub.execute_input":"2022-06-11T10:49:16.456219Z","iopub.status.idle":"2022-06-11T10:49:26.260922Z","shell.execute_reply.started":"2022-06-11T10:49:16.456188Z","shell.execute_reply":"2022-06-11T10:49:26.260025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D_126","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\",\"P_4\",\"D_39\",\"R_27\",\"D_126\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-11T10:49:57.180593Z","iopub.execute_input":"2022-06-11T10:49:57.181610Z","iopub.status.idle":"2022-06-11T10:50:08.065819Z","shell.execute_reply.started":"2022-06-11T10:49:57.181562Z","shell.execute_reply":"2022-06-11T10:50:08.064921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Сonclusion\n","metadata":{}},{"cell_type":"markdown","source":"### Risky features on the last lines: \"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\",\"R_27\",\"S_17\",\"S_13\", \"S_18\",\"D_45\". ","metadata":{}},{"cell_type":"markdown","source":"### Risky features on the first lines \"R_1\",\"D_59\",\"S_11\",\"D_121\",\"S_27\",\"D_118\",\"D_119\",\"D_120\",\"P_4\",\"D_39\",\"R_27\",\"D_126\"","metadata":{}},{"cell_type":"markdown","source":"\n#### You may notice that after the removal of the D_120 feature, roc-auc began to decline more slowly. In general, I decided to stop at 0.67 for the time being, because I still don’t see the point in looking for more features. I think that I will continue to try to carry out the same work on aggregated features","metadata":{}},{"cell_type":"markdown","source":"### Vote for this notebook if its content was useful or interesting to you","metadata":{}}]}