{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cudf\nimport cupy\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom catboost import CatBoost\nfrom catboost import Pool\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom catboost import CatBoostClassifier\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-09T13:54:14.028068Z","iopub.execute_input":"2022-06-09T13:54:14.028505Z","iopub.status.idle":"2022-06-09T13:54:17.874807Z","shell.execute_reply.started":"2022-06-09T13:54:14.028394Z","shell.execute_reply":"2022-06-09T13:54:17.873949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_adversarial_data(df_train, df_test, cols, N_val=70000):\n    df_master = df_train[cols].append(df_test[cols], ignore_index=True)\n    adversarial_val = df_master.sample(N_val, replace=False)\n    adversarial_train = df_master[\n        ~df_master.index.isin(adversarial_val.index)\n    ]\n    return adversarial_train, adversarial_val","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:54:17.876308Z","iopub.execute_input":"2022-06-09T13:54:17.876854Z","iopub.status.idle":"2022-06-09T13:54:17.887613Z","shell.execute_reply.started":"2022-06-09T13:54:17.876816Z","shell.execute_reply":"2022-06-09T13:54:17.886876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This work inspired me to create this notebook - https://www.kaggle.com/code/zakopur0/adversarial-validation-private-vs-public/notebook","metadata":{}},{"cell_type":"markdown","source":"## In this work, I want to find out which features differ the most on the test and training dataset, which can lead to overfitting \n## To do this, I will remove the variables until the ROC-AUC in the classification of the test and the train reaches approximately 0.6","metadata":{}},{"cell_type":"markdown","source":"# Data preparation","metadata":{}},{"cell_type":"code","source":"train = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet').drop_duplicates(subset=[\"customer_ID\"], keep=\"last\")\ntest = cudf.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet').drop_duplicates(subset=[\"customer_ID\"], keep=\"last\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:54:17.889254Z","iopub.execute_input":"2022-06-09T13:54:17.890074Z","iopub.status.idle":"2022-06-09T13:55:28.317790Z","shell.execute_reply.started":"2022-06-09T13:54:17.890032Z","shell.execute_reply":"2022-06-09T13:55:28.316960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"is_train\"] = 1\ntest[\"is_train\"] = 0\ntarget = ['is_train']\ndrop_cols = ['S_2','customer_ID',\"is_train\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:55:28.320233Z","iopub.execute_input":"2022-06-09T13:55:28.320625Z","iopub.status.idle":"2022-06-09T13:55:30.177769Z","shell.execute_reply.started":"2022-06-09T13:55:28.320588Z","shell.execute_reply":"2022-06-09T13:55:30.176822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Starting Negative Feature Selection","metadata":{}},{"cell_type":"code","source":"train_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:55:30.179264Z","iopub.execute_input":"2022-06-09T13:55:30.179692Z","iopub.status.idle":"2022-06-09T13:55:31.810590Z","shell.execute_reply.started":"2022-06-09T13:55:30.179651Z","shell.execute_reply":"2022-06-09T13:55:31.809785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:55:31.811982Z","iopub.execute_input":"2022-06-09T13:55:31.812309Z","iopub.status.idle":"2022-06-09T13:57:05.788242Z","shell.execute_reply.started":"2022-06-09T13:55:31.812274Z","shell.execute_reply":"2022-06-09T13:57:05.787496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:05.792342Z","iopub.execute_input":"2022-06-09T13:57:05.794262Z","iopub.status.idle":"2022-06-09T13:57:06.117509Z","shell.execute_reply.started":"2022-06-09T13:57:05.794225Z","shell.execute_reply":"2022-06-09T13:57:06.116826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop R1","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:06.121213Z","iopub.execute_input":"2022-06-09T13:57:06.123217Z","iopub.status.idle":"2022-06-09T13:57:17.106254Z","shell.execute_reply.started":"2022-06-09T13:57:06.123178Z","shell.execute_reply":"2022-06-09T13:57:17.105500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D59","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:17.109876Z","iopub.execute_input":"2022-06-09T13:57:17.111782Z","iopub.status.idle":"2022-06-09T13:57:27.879202Z","shell.execute_reply.started":"2022-06-09T13:57:17.111745Z","shell.execute_reply":"2022-06-09T13:57:27.878499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S11","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:27.884450Z","iopub.execute_input":"2022-06-09T13:57:27.886310Z","iopub.status.idle":"2022-06-09T13:57:38.360238Z","shell.execute_reply.started":"2022-06-09T13:57:27.886274Z","shell.execute_reply":"2022-06-09T13:57:38.359507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop B29","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:38.364189Z","iopub.execute_input":"2022-06-09T13:57:38.366144Z","iopub.status.idle":"2022-06-09T13:57:48.816727Z","shell.execute_reply.started":"2022-06-09T13:57:38.366089Z","shell.execute_reply":"2022-06-09T13:57:48.816038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S9","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:48.820169Z","iopub.execute_input":"2022-06-09T13:57:48.821048Z","iopub.status.idle":"2022-06-09T13:57:59.544980Z","shell.execute_reply.started":"2022-06-09T13:57:48.821000Z","shell.execute_reply":"2022-06-09T13:57:59.544253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S15","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:57:59.548513Z","iopub.execute_input":"2022-06-09T13:57:59.549256Z","iopub.status.idle":"2022-06-09T13:58:09.886202Z","shell.execute_reply.started":"2022-06-09T13:57:59.549219Z","shell.execute_reply":"2022-06-09T13:58:09.885488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D121","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:58:09.889927Z","iopub.execute_input":"2022-06-09T13:58:09.890703Z","iopub.status.idle":"2022-06-09T13:58:20.097047Z","shell.execute_reply.started":"2022-06-09T13:58:09.890666Z","shell.execute_reply":"2022-06-09T13:58:20.096378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S24","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:58:20.100710Z","iopub.execute_input":"2022-06-09T13:58:20.102603Z","iopub.status.idle":"2022-06-09T13:58:31.293957Z","shell.execute_reply.started":"2022-06-09T13:58:20.102568Z","shell.execute_reply":"2022-06-09T13:58:31.293268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop D62","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:58:31.297780Z","iopub.execute_input":"2022-06-09T13:58:31.298452Z","iopub.status.idle":"2022-06-09T13:58:41.525221Z","shell.execute_reply.started":"2022-06-09T13:58:31.298397Z","shell.execute_reply":"2022-06-09T13:58:41.524496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop R27","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\",\"R_27\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:58:41.529083Z","iopub.execute_input":"2022-06-09T13:58:41.531099Z","iopub.status.idle":"2022-06-09T13:58:51.578004Z","shell.execute_reply.started":"2022-06-09T13:58:41.531063Z","shell.execute_reply":"2022-06-09T13:58:51.577312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop S17","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\",\"R_27\",\"S_17\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:58:51.581709Z","iopub.execute_input":"2022-06-09T13:58:51.583577Z","iopub.status.idle":"2022-06-09T13:59:02.422003Z","shell.execute_reply.started":"2022-06-09T13:58:51.583540Z","shell.execute_reply":"2022-06-09T13:59:02.421264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop \"S_13\", \"S_18\",\"D_45\"","metadata":{}},{"cell_type":"code","source":"drop_cols = ['S_2','customer_ID',\"is_train\",\"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\",\"R_27\",\"S_17\",\"S_13\", \"S_18\",\"D_45\"]\nuse_cols = [c for c in train.columns if c not in drop_cols]\n            \n    \nadversarial_train, adversarial_test = create_adversarial_data(train, test, list(train.columns))\ntrain_data = Pool(\n    data=adversarial_train[use_cols].to_pandas(),\n    label=adversarial_train[target].to_pandas()\n)\nholdout_data = Pool(\n    data=adversarial_test[use_cols].to_pandas(),\n    label=adversarial_test[target].to_pandas()\n)\nparams = {\n    'loss_function' : 'Logloss',\n    'eval_metric' : 'AUC',\n    'od_type': 'Iter',\n    'num_boost_round': 100,\n    'early_stopping_rounds': 10,\n    \"depth\": 2,\n    'task_type': 'GPU'\n}\n\nmodel = CatBoostClassifier(**params)\n_ = model.fit(train_data, eval_set=holdout_data, verbose_eval=50)\n\nfeature_importance =pd.DataFrame({'feature_importance': model.get_feature_importance(), \n              'feature_names': adversarial_train[use_cols].columns}).sort_values(by=['feature_importance'], \n                                                           ascending=False)\nf, ax = plt.subplots(figsize=(10, 10))\nsns.set_color_codes(\"pastel\")\nsns.barplot(x=\"feature_names\", y=\"feature_importance\", data=feature_importance.sort_values(by=\"feature_importance\", ascending=False)[:10],color=\"b\")","metadata":{"execution":{"iopub.status.busy":"2022-06-09T13:59:02.426596Z","iopub.execute_input":"2022-06-09T13:59:02.428704Z","iopub.status.idle":"2022-06-09T13:59:12.530733Z","shell.execute_reply.started":"2022-06-09T13:59:02.428665Z","shell.execute_reply":"2022-06-09T13:59:12.529964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Сonclusion\n","metadata":{}},{"cell_type":"markdown","source":"### When training the model, you need to be careful with the variables: \"R_1\",\"D_59\",\"S_11\",\"B_29\",\"S_9\",\"S_15\",\"D_121\",\"S_24\", \"D_62\",\"R_27\",\"S_17\",\"S_13\", \"S_18\",\"D_45\". Because they can lead to overfitting","metadata":{}},{"cell_type":"markdown","source":"### Vote for this notebook if its content was useful or interesting to you","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}