{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-`1qonly \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-22T16:19:36.785868Z","iopub.execute_input":"2023-01-22T16:19:36.786312Z","iopub.status.idle":"2023-01-22T16:19:36.799662Z","shell.execute_reply.started":"2023-01-22T16:19:36.786279Z","shell.execute_reply":"2023-01-22T16:19:36.798500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nimport os\n\n\nimport lightgbm\nfrom xgboost import XGBClassifier\n\nimport matplotlib.pyplot as plt\nimport seaborn as sbn\nimport numpy as np\nimport pandas as pd\nimport warnings\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import confusion_matrix, classification_report, roc_curve, auc, RocCurveDisplay, accuracy_score\n\nimport shap\nimport itertools","metadata":{"execution":{"iopub.status.busy":"2023-01-22T16:19:44.619163Z","iopub.execute_input":"2023-01-22T16:19:44.619551Z","iopub.status.idle":"2023-01-22T16:19:48.769373Z","shell.execute_reply.started":"2023-01-22T16:19:44.619520Z","shell.execute_reply":"2023-01-22T16:19:48.768054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ntest = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet\")\ntrain_labels = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\n#submission = pd.read_csv(\"/kaggle/input/amex-default-prediction/sample_submission.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2023-01-22T16:19:50.177454Z","iopub.execute_input":"2023-01-22T16:19:50.178203Z","iopub.status.idle":"2023-01-22T16:21:10.262417Z","shell.execute_reply.started":"2023-01-22T16:19:50.178165Z","shell.execute_reply":"2023-01-22T16:21:10.259591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Describe","metadata":{}},{"cell_type":"code","source":"train.shape, test.shape, train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T08:07:56.920553Z","iopub.execute_input":"2023-01-17T08:07:56.921294Z","iopub.status.idle":"2023-01-17T08:07:56.937022Z","shell.execute_reply.started":"2023-01-17T08:07:56.921223Z","shell.execute_reply":"2023-01-17T08:07:56.934690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.duplicated().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns.values:\n  print(col,'-',train[col].isna().sum()/len(train[col]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_numeric_cols = train.columns[train.dtypes == 'object'].values\nnon_numeric_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = train.columns[train.dtypes != 'object'].values\nnumeric_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# for i in numeric_cols:\n#     X = train[i].round(decimals = 3)\n#     plt.figure(i)\n#     ax = sns.countplot(X)\n#     ax.set_xticklabels(ax.get_xticklabels(), rotation=40, fontsize=5)\n#     plt.tight_layout()\n#     plt.show()        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"import matplotlib.style as style\nimport seaborn as sns\nstyle.use('seaborn-poster')\nsns.set_style('ticks')\nplt.subplots(figsize = (270,200))\n## Plotting heatmap. \n\n# Generate a mask for the upper triangle (taken from seaborn example gallery)\nmask = np.zeros_like(train.corr(), dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n\nsns.heatmap(train.corr(), cmap=plt.get_cmap('Blues'), annot=True, mask=mask, center = 0, square=True, \n             );\n## Give title. \nplt.title(\"Heatmap of all the Features\", fontsize = 25);","metadata":{"execution":{"iopub.status.busy":"2023-01-22T16:21:10.269380Z","iopub.execute_input":"2023-01-22T16:21:10.269934Z","iopub.status.idle":"2023-01-22T16:39:11.283331Z","shell.execute_reply.started":"2023-01-22T16:21:10.269880Z","shell.execute_reply":"2023-01-22T16:39:11.281708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Train","metadata":{}},{"cell_type":"code","source":"train['Date'] =  pd.to_datetime(train['S_2'], format=\"%Y/%m/%d\")\ntrain['weekday'] = train['Date'].dt.weekday\ntrain['day'] = train['Date'].dt.day\ntrain['month'] = train['Date'].dt.month\ntrain['year'] = train['Date'].dt.year","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:37:22.087928Z","iopub.execute_input":"2023-01-22T13:37:22.088390Z","iopub.status.idle":"2023-01-22T13:37:23.601821Z","shell.execute_reply.started":"2023-01-22T13:37:22.088358Z","shell.execute_reply":"2023-01-22T13:37:23.600345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['S_2'] = pd.to_numeric(train['S_2'].str.replace('-',''))\ntrain['S_2']","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:37:23.603435Z","iopub.execute_input":"2023-01-22T13:37:23.603881Z","iopub.status.idle":"2023-01-22T13:37:29.113464Z","shell.execute_reply.started":"2023-01-22T13:37:23.603841Z","shell.execute_reply":"2023-01-22T13:37:29.112056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['Date'], axis=1, inplace=True)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:37:29.116291Z","iopub.execute_input":"2023-01-22T13:37:29.116735Z","iopub.status.idle":"2023-01-22T13:37:30.653315Z","shell.execute_reply.started":"2023-01-22T13:37:29.116699Z","shell.execute_reply":"2023-01-22T13:37:30.652405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",'customer_ID']","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:32.534795Z","iopub.execute_input":"2023-01-22T13:39:32.535255Z","iopub.status.idle":"2023-01-22T13:39:32.541913Z","shell.execute_reply.started":"2023-01-22T13:39:32.535221Z","shell.execute_reply":"2023-01-22T13:39:32.540501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in train.columns.values if col not in cat_features]\nfeatures.append('customer_ID')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:39.969455Z","iopub.execute_input":"2023-01-22T13:39:39.969829Z","iopub.status.idle":"2023-01-22T13:39:39.977112Z","shell.execute_reply.started":"2023-01-22T13:39:39.969799Z","shell.execute_reply":"2023-01-22T13:39:39.975558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat_features:\n    if train[i].dtype == 'int64':\n        train.astype('int16')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:41.033354Z","iopub.execute_input":"2023-01-22T13:39:41.033727Z","iopub.status.idle":"2023-01-22T13:39:41.040452Z","shell.execute_reply.started":"2023-01-22T13:39:41.033697Z","shell.execute_reply":"2023-01-22T13:39:41.039365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cat = train[cat_features].groupby('customer_ID',as_index=False).agg(['last', 'nunique'])\ntrain_cat.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:44.043304Z","iopub.execute_input":"2023-01-22T13:39:44.043728Z","iopub.status.idle":"2023-01-22T13:39:49.781468Z","shell.execute_reply.started":"2023-01-22T13:39:44.043696Z","shell.execute_reply":"2023-01-22T13:39:49.780047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train.columns:\n    if train[i].dtype == 'float64':\n        train.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:52.743001Z","iopub.execute_input":"2023-01-22T13:39:52.743410Z","iopub.status.idle":"2023-01-22T13:39:52.757497Z","shell.execute_reply.started":"2023-01-22T13:39:52.743377Z","shell.execute_reply":"2023-01-22T13:39:52.755394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:55.153184Z","iopub.execute_input":"2023-01-22T13:39:55.153589Z","iopub.status.idle":"2023-01-22T13:39:55.159327Z","shell.execute_reply.started":"2023-01-22T13:39:55.153556Z","shell.execute_reply":"2023-01-22T13:39:55.157709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(drop_features, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:39:58.012906Z","iopub.execute_input":"2023-01-22T13:39:58.013338Z","iopub.status.idle":"2023-01-22T13:39:59.466486Z","shell.execute_reply.started":"2023-01-22T13:39:58.013304Z","shell.execute_reply":"2023-01-22T13:39:59.465049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.groupby('customer_ID',as_index=False).agg(['mean', 'std', 'sum','last'])\n\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:40:06.729100Z","iopub.execute_input":"2023-01-22T13:40:06.729465Z","iopub.status.idle":"2023-01-22T13:41:01.817192Z","shell.execute_reply.started":"2023-01-22T13:40:06.729436Z","shell.execute_reply":"2023-01-22T13:41:01.815913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:41:01.819319Z","iopub.execute_input":"2023-01-22T13:41:01.819681Z","iopub.status.idle":"2023-01-22T13:41:02.071042Z","shell.execute_reply.started":"2023-01-22T13:41:01.819651Z","shell.execute_reply":"2023-01-22T13:41:02.069673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(train_cat, how='inner', on=\"customer_ID\")\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:41:33.033868Z","iopub.execute_input":"2023-01-22T13:41:33.034325Z","iopub.status.idle":"2023-01-22T13:44:05.962398Z","shell.execute_reply.started":"2023-01-22T13:41:33.034291Z","shell.execute_reply":"2023-01-22T13:44:05.961121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_cat","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:48:55.764712Z","iopub.execute_input":"2023-01-22T13:48:55.765224Z","iopub.status.idle":"2023-01-22T13:48:55.812157Z","shell.execute_reply.started":"2023-01-22T13:48:55.765192Z","shell.execute_reply":"2023-01-22T13:48:55.810689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_col = []\nfor i in train.columns.values:\n    if type(i) is tuple:\n        col = '_'.join(i)\n        join_col.append(col)\ntrain.columns = join_col\ntrain.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:49:00.312975Z","iopub.execute_input":"2023-01-22T13:49:00.313621Z","iopub.status.idle":"2023-01-22T13:49:02.739380Z","shell.execute_reply.started":"2023-01-22T13:49:00.313587Z","shell.execute_reply":"2023-01-22T13:49:02.737846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(train_labels, how='inner', on=\"customer_ID\")\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:49:14.542905Z","iopub.execute_input":"2023-01-22T13:49:14.543319Z","iopub.status.idle":"2023-01-22T13:49:16.400235Z","shell.execute_reply.started":"2023-01-22T13:49:14.543290Z","shell.execute_reply":"2023-01-22T13:49:16.398813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_matrix = train.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n# Find features with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] >= 0.95)]\n\n# Drop features \n\nto_drop","metadata":{"execution":{"iopub.status.busy":"2023-01-22T13:49:24.063160Z","iopub.execute_input":"2023-01-22T13:49:24.063522Z","iopub.status.idle":"2023-01-22T13:56:37.131318Z","shell.execute_reply.started":"2023-01-22T13:49:24.063492Z","shell.execute_reply":"2023-01-22T13:56:37.129575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(to_drop)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:02:39.874503Z","iopub.execute_input":"2023-01-17T15:02:39.875341Z","iopub.status.idle":"2023-01-17T15:02:39.886863Z","shell.execute_reply.started":"2023-01-17T15:02:39.875287Z","shell.execute_reply":"2023-01-17T15:02:39.885091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(to_drop, axis=1, inplace=True)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:06:06.510539Z","iopub.execute_input":"2023-01-22T14:06:06.511409Z","iopub.status.idle":"2023-01-22T14:06:06.685080Z","shell.execute_reply.started":"2023-01-22T14:06:06.511365Z","shell.execute_reply":"2023-01-22T14:06:06.683687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Test","metadata":{}},{"cell_type":"code","source":"test['Date'] =  pd.to_datetime(test['S_2'], format=\"%Y/%m/%d\")\ntest['weekday'] = test['Date'].dt.weekday\ntest['day'] = test['Date'].dt.day\ntest['month'] = test['Date'].dt.month\ntest['year'] = test['Date'].dt.year","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:02:43.314016Z","iopub.execute_input":"2023-01-17T15:02:43.314410Z","iopub.status.idle":"2023-01-17T15:02:49.299189Z","shell.execute_reply.started":"2023-01-17T15:02:43.314375Z","shell.execute_reply":"2023-01-17T15:02:49.297886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['S_2'] = pd.to_numeric(test['S_2'].str.replace('-',''))\ntest['S_2']","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:02:49.301324Z","iopub.execute_input":"2023-01-17T15:02:49.301758Z","iopub.status.idle":"2023-01-17T15:03:02.740610Z","shell.execute_reply.started":"2023-01-17T15:02:49.301719Z","shell.execute_reply":"2023-01-17T15:03:02.739281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['Date'], axis=1, inplace=True)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:02.742562Z","iopub.execute_input":"2023-01-17T15:03:02.743966Z","iopub.status.idle":"2023-01-17T15:03:10.164715Z","shell.execute_reply.started":"2023-01-17T15:03:02.743912Z","shell.execute_reply":"2023-01-17T15:03:10.163355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\",'customer_ID']","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:10.166600Z","iopub.execute_input":"2023-01-17T15:03:10.167020Z","iopub.status.idle":"2023-01-17T15:03:10.173538Z","shell.execute_reply.started":"2023-01-17T15:03:10.166981Z","shell.execute_reply":"2023-01-17T15:03:10.172243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in test.columns.values:\n    if test[i].dtype == 'int64' and i == 'customer_ID':\n        test.astype('int16')","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:10.175311Z","iopub.execute_input":"2023-01-17T15:03:10.175764Z","iopub.status.idle":"2023-01-17T15:03:10.590442Z","shell.execute_reply.started":"2023-01-17T15:03:10.175724Z","shell.execute_reply":"2023-01-17T15:03:10.588950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:10.592158Z","iopub.execute_input":"2023-01-17T15:03:10.593095Z","iopub.status.idle":"2023-01-17T15:03:10.601968Z","shell.execute_reply.started":"2023-01-17T15:03:10.593050Z","shell.execute_reply":"2023-01-17T15:03:10.600952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_copy = test[cat_features].groupby('customer_ID',as_index=False).agg([ 'last', 'nunique'])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:10.603626Z","iopub.execute_input":"2023-01-17T15:03:10.604832Z","iopub.status.idle":"2023-01-17T15:03:36.186478Z","shell.execute_reply.started":"2023-01-17T15:03:10.604752Z","shell.execute_reply":"2023-01-17T15:03:36.185119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in test.columns.values:\n    if test[i].dtype == 'float64':\n        test.astype('float16')","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:36.188167Z","iopub.execute_input":"2023-01-17T15:03:36.189080Z","iopub.status.idle":"2023-01-17T15:03:36.197291Z","shell.execute_reply.started":"2023-01-17T15:03:36.189031Z","shell.execute_reply":"2023-01-17T15:03:36.196204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drp_col = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:36.207308Z","iopub.execute_input":"2023-01-17T15:03:36.208095Z","iopub.status.idle":"2023-01-17T15:03:36.214762Z","shell.execute_reply.started":"2023-01-17T15:03:36.208053Z","shell.execute_reply":"2023-01-17T15:03:36.213339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(drp_col, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:36.216615Z","iopub.execute_input":"2023-01-17T15:03:36.217250Z","iopub.status.idle":"2023-01-17T15:03:43.821058Z","shell.execute_reply.started":"2023-01-17T15:03:36.217198Z","shell.execute_reply":"2023-01-17T15:03:43.819806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:43.822810Z","iopub.execute_input":"2023-01-17T15:03:43.823336Z","iopub.status.idle":"2023-01-17T15:03:44.028719Z","shell.execute_reply.started":"2023-01-17T15:03:43.823284Z","shell.execute_reply":"2023-01-17T15:03:44.027344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.groupby('customer_ID',as_index=False).agg(['mean', 'std', 'sum','last'])\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:03:44.031069Z","iopub.execute_input":"2023-01-17T15:03:44.031515Z","iopub.status.idle":"2023-01-17T15:06:21.904133Z","shell.execute_reply.started":"2023-01-17T15:03:44.031477Z","shell.execute_reply":"2023-01-17T15:06:21.902709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:06:21.905999Z","iopub.execute_input":"2023-01-17T15:06:21.906406Z","iopub.status.idle":"2023-01-17T15:06:21.914911Z","shell.execute_reply.started":"2023-01-17T15:06:21.906357Z","shell.execute_reply":"2023-01-17T15:06:21.913701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.merge(test_copy, how='inner', on=\"customer_ID\")\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:06:21.916838Z","iopub.execute_input":"2023-01-17T15:06:21.917216Z","iopub.status.idle":"2023-01-17T15:12:04.868728Z","shell.execute_reply.started":"2023-01-17T15:06:21.917186Z","shell.execute_reply":"2023-01-17T15:12:04.867043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_col = []\nfor i in test.columns.values:\n    if type(i) is tuple:\n        col = '_'.join(i)\n        join_col.append(col)\n        \ntest.columns = join_col\ntest.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:04.871003Z","iopub.execute_input":"2023-01-17T15:12:04.871469Z","iopub.status.idle":"2023-01-17T15:12:09.844496Z","shell.execute_reply.started":"2023-01-17T15:12:04.871432Z","shell.execute_reply":"2023-01-17T15:12:09.842813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(to_drop, axis=1, inplace=True)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:09.847136Z","iopub.execute_input":"2023-01-17T15:12:09.848660Z","iopub.status.idle":"2023-01-17T15:12:14.934884Z","shell.execute_reply.started":"2023-01-17T15:12:09.848598Z","shell.execute_reply":"2023-01-17T15:12:14.933839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_copy\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:14.936958Z","iopub.execute_input":"2023-01-17T15:12:14.937815Z","iopub.status.idle":"2023-01-17T15:12:14.958059Z","shell.execute_reply.started":"2023-01-17T15:12:14.937756Z","shell.execute_reply":"2023-01-17T15:12:14.956461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:14.960169Z","iopub.execute_input":"2023-01-17T15:12:14.960744Z","iopub.status.idle":"2023-01-17T15:12:15.263811Z","shell.execute_reply.started":"2023-01-17T15:12:14.960686Z","shell.execute_reply":"2023-01-17T15:12:15.262469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"target = train['target']\nFeatures = train.drop('target', axis=1, inplace=False)\ntrain.shape, Features.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:06:16.490900Z","iopub.execute_input":"2023-01-22T14:06:16.491854Z","iopub.status.idle":"2023-01-22T14:06:16.744709Z","shell.execute_reply.started":"2023-01-22T14:06:16.491801Z","shell.execute_reply":"2023-01-22T14:06:16.743539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = Features.columns[Features.dtypes != \"object\"].values\nnon_numeric_cols = Features.columns[Features.dtypes == 'object'].values","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:06:23.614629Z","iopub.execute_input":"2023-01-22T14:06:23.615035Z","iopub.status.idle":"2023-01-22T14:06:23.623150Z","shell.execute_reply.started":"2023-01-22T14:06:23.614984Z","shell.execute_reply":"2023-01-22T14:06:23.621813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_1 = test.loc[:, Features.columns]\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:16.222475Z","iopub.execute_input":"2023-01-17T15:12:16.223147Z","iopub.status.idle":"2023-01-17T15:12:16.234343Z","shell.execute_reply.started":"2023-01-17T15:12:16.223041Z","shell.execute_reply":"2023-01-17T15:12:16.232698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Features.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:16.238550Z","iopub.execute_input":"2023-01-17T15:12:16.239041Z","iopub.status.idle":"2023-01-17T15:12:16.250955Z","shell.execute_reply.started":"2023-01-17T15:12:16.239000Z","shell.execute_reply":"2023-01-17T15:12:16.249814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\n\nfrom sklearn.pipeline import Pipeline\n\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder, OrdinalEncoder","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:06:39.249721Z","iopub.execute_input":"2023-01-22T14:06:39.250162Z","iopub.status.idle":"2023-01-22T14:06:39.267806Z","shell.execute_reply.started":"2023-01-22T14:06:39.250120Z","shell.execute_reply":"2023-01-22T14:06:39.266764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnumeric_preprocessing_steps = Pipeline(steps=[\n    ('standard_scaler', StandardScaler()),\n    ('imputer', SimpleImputer(strategy='mean')),\n    ])\n\nnon_numeric_preprocessing_steps = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n    #('onehot', OrdinalEncoder())\n    ('onehot', OneHotEncoder(handle_unknown='ignore'))\n    ])\n\n\npreprocessor = ColumnTransformer(\n    transformers = [\n        (\"numeric\", numeric_preprocessing_steps, numeric_cols),\n        #(\"non_numeric\",non_numeric_preprocessing_steps,non_numeric_cols)\n    ],\n    remainder='drop'\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:06:41.897727Z","iopub.execute_input":"2023-01-22T14:06:41.898629Z","iopub.status.idle":"2023-01-22T14:06:41.904443Z","shell.execute_reply.started":"2023-01-22T14:06:41.898595Z","shell.execute_reply":"2023-01-22T14:06:41.903573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing score","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_eval, y_train, y_eval = train_test_split(  \n    Features,\n    train['target'],\n    test_size=0.2,\n    shuffle=True,\n    random_state=8\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:08:25.437074Z","iopub.execute_input":"2023-01-22T14:08:25.437487Z","iopub.status.idle":"2023-01-22T14:08:28.065830Z","shell.execute_reply.started":"2023-01-22T14:08:25.437457Z","shell.execute_reply":"2023-01-22T14:08:28.064813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    \n    \n    for i in [1,0]:\n        \n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:08:30.643445Z","iopub.execute_input":"2023-01-22T14:08:30.643776Z","iopub.status.idle":"2023-01-22T14:08:30.656523Z","shell.execute_reply.started":"2023-01-22T14:08:30.643751Z","shell.execute_reply":"2023-01-22T14:08:30.654640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KNN","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\n\nKNN = KNeighborsClassifier(15)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.825585Z","iopub.execute_input":"2023-01-17T15:12:19.826164Z","iopub.status.idle":"2023-01-17T15:12:19.842763Z","shell.execute_reply.started":"2023-01-17T15:12:19.826098Z","shell.execute_reply":"2023-01-17T15:12:19.841438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline = Pipeline([\n    (\"preprocessor\", preprocessor),\n    (\"estimators\", KNN),\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.844563Z","iopub.execute_input":"2023-01-17T15:12:19.844990Z","iopub.status.idle":"2023-01-17T15:12:19.860777Z","shell.execute_reply.started":"2023-01-17T15:12:19.844955Z","shell.execute_reply":"2023-01-17T15:12:19.858159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#full_pipeline.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.862274Z","iopub.execute_input":"2023-01-17T15:12:19.863216Z","iopub.status.idle":"2023-01-17T15:12:19.877767Z","shell.execute_reply.started":"2023-01-17T15:12:19.863135Z","shell.execute_reply":"2023-01-17T15:12:19.875935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_pred = full_pipeline.predict_proba(X_eval)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.879854Z","iopub.execute_input":"2023-01-17T15:12:19.880322Z","iopub.status.idle":"2023-01-17T15:12:19.892445Z","shell.execute_reply.started":"2023-01-17T15:12:19.880280Z","shell.execute_reply":"2023-01-17T15:12:19.890907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#amex_metric_mod(y_eval.values, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.893999Z","iopub.execute_input":"2023-01-17T15:12:19.894476Z","iopub.status.idle":"2023-01-17T15:12:19.908351Z","shell.execute_reply.started":"2023-01-17T15:12:19.894433Z","shell.execute_reply":"2023-01-17T15:12:19.907227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n#roc_auc_score( y_eval, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.910178Z","iopub.execute_input":"2023-01-17T15:12:19.911465Z","iopub.status.idle":"2023-01-17T15:12:19.926315Z","shell.execute_reply.started":"2023-01-17T15:12:19.911410Z","shell.execute_reply":"2023-01-17T15:12:19.924758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SVM","metadata":{}},{"cell_type":"code","source":"from sklearn.svm import SVC\n\nsvm = SVC(kernel='linear')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:38:54.420318Z","iopub.execute_input":"2023-01-22T14:38:54.420730Z","iopub.status.idle":"2023-01-22T14:38:54.426627Z","shell.execute_reply.started":"2023-01-22T14:38:54.420700Z","shell.execute_reply":"2023-01-22T14:38:54.425493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline = Pipeline([\n    (\"preprocessor\", preprocessor),\n    (\"estimators\", svm),\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:39:01.938547Z","iopub.execute_input":"2023-01-22T14:39:01.938977Z","iopub.status.idle":"2023-01-22T14:39:01.945829Z","shell.execute_reply.started":"2023-01-22T14:39:01.938942Z","shell.execute_reply":"2023-01-22T14:39:01.943931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#full_pipeline.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:40:00.079238Z","iopub.execute_input":"2023-01-22T14:40:00.080637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn import metrics \n\n# predicted = full_pipeline.predict(X_eval)\n# confusion_matrix = metrics.confusion_matrix(y_eval, predicted) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# sns.heatmap(confusion_matrix/np.sum(confusion_matrix), annot=True,fmt='.2%', cmap='Blues')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_pred = full_pipeline.predict_proba(X_eval)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.975985Z","iopub.execute_input":"2023-01-17T15:12:19.976488Z","iopub.status.idle":"2023-01-17T15:12:19.990480Z","shell.execute_reply.started":"2023-01-17T15:12:19.976438Z","shell.execute_reply":"2023-01-17T15:12:19.988863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#amex_metric_mod(y_eval.values, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:19.992195Z","iopub.execute_input":"2023-01-17T15:12:19.993181Z","iopub.status.idle":"2023-01-17T15:12:20.005685Z","shell.execute_reply.started":"2023-01-17T15:12:19.993130Z","shell.execute_reply":"2023-01-17T15:12:20.003675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n#roc_auc_score( y_eval, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:12:20.007002Z","iopub.execute_input":"2023-01-17T15:12:20.007633Z","iopub.status.idle":"2023-01-17T15:12:20.022244Z","shell.execute_reply.started":"2023-01-17T15:12:20.007580Z","shell.execute_reply":"2023-01-17T15:12:20.020442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nlgbm = LGBMClassifier(\n    n_estimators= 3000, \n    num_leaves= 100,\n    learning_rate= 0.01,\n    colsample_bytree= 0.6,\n    objective = 'binary',\n    max_depth= 8,\n    min_data_in_leaf = 27,\n    bagging_freq = 7,\n    bagging_fraction= 0.8,\n    feature_fraction = 0.4,\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:08:53.394959Z","iopub.execute_input":"2023-01-22T14:08:53.395338Z","iopub.status.idle":"2023-01-22T14:08:53.400889Z","shell.execute_reply.started":"2023-01-22T14:08:53.395309Z","shell.execute_reply":"2023-01-22T14:08:53.400089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline = Pipeline([\n    (\"preprocessor\", preprocessor),\n    (\"estimators\", lgbm),\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:08:59.114817Z","iopub.execute_input":"2023-01-22T14:08:59.115465Z","iopub.status.idle":"2023-01-22T14:08:59.121174Z","shell.execute_reply.started":"2023-01-22T14:08:59.115403Z","shell.execute_reply":"2023-01-22T14:08:59.119714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_pipeline.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:09:11.675855Z","iopub.execute_input":"2023-01-22T14:09:11.676825Z","iopub.status.idle":"2023-01-22T14:24:26.931774Z","shell.execute_reply.started":"2023-01-22T14:09:11.676787Z","shell.execute_reply":"2023-01-22T14:24:26.930184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_pred = full_pipeline.predict_proba(X_eval)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:28:35.092574Z","iopub.execute_input":"2023-01-17T15:28:35.093812Z","iopub.status.idle":"2023-01-17T15:29:05.623902Z","shell.execute_reply.started":"2023-01-17T15:28:35.093752Z","shell.execute_reply":"2023-01-17T15:29:05.622504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#amex_metric_mod(y_eval.values, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:05.625865Z","iopub.execute_input":"2023-01-17T15:29:05.626335Z","iopub.status.idle":"2023-01-17T15:29:05.701191Z","shell.execute_reply.started":"2023-01-17T15:29:05.626296Z","shell.execute_reply":"2023-01-17T15:29:05.699912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn import metrics \n\n# predicted = full_pipeline.predict(X_eval)\n# confusion_matrix = metrics.confusion_matrix(y_eval, predicted) ","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:24:26.934153Z","iopub.execute_input":"2023-01-22T14:24:26.934566Z","iopub.status.idle":"2023-01-22T14:24:42.844346Z","shell.execute_reply.started":"2023-01-22T14:24:26.934530Z","shell.execute_reply":"2023-01-22T14:24:42.843077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# sns.heatmap(confusion_matrix/np.sum(confusion_matrix), annot=True,fmt='.2%', cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T14:37:25.127938Z","iopub.execute_input":"2023-01-22T14:37:25.128324Z","iopub.status.idle":"2023-01-22T14:37:25.334367Z","shell.execute_reply.started":"2023-01-22T14:37:25.128300Z","shell.execute_reply":"2023-01-22T14:37:25.333072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import roc_auc_score\n\n# roc_auc_score( y_eval, y_pred[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:05.702623Z","iopub.execute_input":"2023-01-17T15:29:05.702987Z","iopub.status.idle":"2023-01-17T15:29:05.769578Z","shell.execute_reply.started":"2023-01-17T15:29:05.702956Z","shell.execute_reply":"2023-01-17T15:29:05.768242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stacked","metadata":{}},{"cell_type":"code","source":"XGB = XGBClassifier(n_estimators=300, max_depth=6, learning_rate=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:05.771117Z","iopub.execute_input":"2023-01-17T15:29:05.771564Z","iopub.status.idle":"2023-01-17T15:29:05.779855Z","shell.execute_reply.started":"2023-01-17T15:29:05.771528Z","shell.execute_reply":"2023-01-17T15:29:05.778199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:05.781963Z","iopub.execute_input":"2023-01-17T15:29:05.782430Z","iopub.status.idle":"2023-01-17T15:29:06.071871Z","shell.execute_reply.started":"2023-01-17T15:29:05.782391Z","shell.execute_reply":"2023-01-17T15:29:06.070712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat = CatBoostClassifier(iterations=1800, random_state=22,\n                         learning_rate=0.03,\n                         max_depth=9,\n                         objective='Logloss',\n                         subsample = 0.4,\n                         colsample_bylevel=0.3\n                        )","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.073564Z","iopub.execute_input":"2023-01-17T15:29:06.073941Z","iopub.status.idle":"2023-01-17T15:29:06.083385Z","shell.execute_reply.started":"2023-01-17T15:29:06.073909Z","shell.execute_reply":"2023-01-17T15:29:06.082090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\nlgbm = LGBMClassifier(\n\n    n_estimators= 3000, \n    num_leaves= 100,\n    learning_rate= 0.01,\n    colsample_bytree= 0.6,\n    objective = 'binary',\n    max_depth= 8,\n    min_data_in_leaf = 27,\n    bagging_freq = 7,\n    bagging_fraction= 0.8,\n    feature_fraction = 0.4,\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.085283Z","iopub.execute_input":"2023-01-17T15:29:06.085853Z","iopub.status.idle":"2023-01-17T15:29:06.097347Z","shell.execute_reply.started":"2023-01-17T15:29:06.085804Z","shell.execute_reply":"2023-01-17T15:29:06.095692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import StackingClassifier\nfrom sklearn.linear_model import LogisticRegression\n\nestimators_stacked = [\n            \n             ('cbc', cat),\n            ('lgbm', lgbm),\n              #('xbg', XGB)\n]    \n        \n\n\nstacked_estimator =  StackingClassifier(estimators=estimators_stacked,\n                                    final_estimator= LogisticRegression(),\n                                    stack_method='predict_proba'\n                                  )","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.098844Z","iopub.execute_input":"2023-01-17T15:29:06.099218Z","iopub.status.idle":"2023-01-17T15:29:06.110641Z","shell.execute_reply.started":"2023-01-17T15:29:06.099185Z","shell.execute_reply":"2023-01-17T15:29:06.109480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline = Pipeline([\n    (\"preprocessor\", preprocessor),\n    (\"estimators\", stacked_estimator),\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.111747Z","iopub.execute_input":"2023-01-17T15:29:06.112103Z","iopub.status.idle":"2023-01-17T15:29:06.125994Z","shell.execute_reply.started":"2023-01-17T15:29:06.112072Z","shell.execute_reply":"2023-01-17T15:29:06.124661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.127417Z","iopub.execute_input":"2023-01-17T15:29:06.127801Z","iopub.status.idle":"2023-01-17T15:29:06.339492Z","shell.execute_reply.started":"2023-01-17T15:29:06.127767Z","shell.execute_reply":"2023-01-17T15:29:06.338065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Output","metadata":{}},{"cell_type":"code","source":"full_pipeline.fit(Features,target)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:29:06.341430Z","iopub.execute_input":"2023-01-17T15:29:06.341969Z","iopub.status.idle":"2023-01-17T15:51:01.325347Z","shell.execute_reply.started":"2023-01-17T15:29:06.341929Z","shell.execute_reply":"2023-01-17T15:51:01.323739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#import numpy as np\n#np.mean(cross_val_score(full_pipeline, Features, target, scoring='accuracy', cv=5))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:01.327421Z","iopub.execute_input":"2023-01-17T15:51:01.327934Z","iopub.status.idle":"2023-01-17T15:51:01.333834Z","shell.execute_reply.started":"2023-01-17T15:51:01.327891Z","shell.execute_reply":"2023-01-17T15:51:01.332349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n\njoblib.dump(full_pipeline, 'pipes.joblib')","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:01.335608Z","iopub.execute_input":"2023-01-17T15:51:01.336059Z","iopub.status.idle":"2023-01-17T15:51:02.370513Z","shell.execute_reply.started":"2023-01-17T15:51:01.336020Z","shell.execute_reply":"2023-01-17T15:51:02.369139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nfull_pipeline = joblib.load('pipes.joblib')","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:02.372680Z","iopub.execute_input":"2023-01-17T15:51:02.373085Z","iopub.status.idle":"2023-01-17T15:51:02.583532Z","shell.execute_reply.started":"2023-01-17T15:51:02.373051Z","shell.execute_reply":"2023-01-17T15:51:02.582189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ndel Features\ndel X_train, X_eval\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:02.585337Z","iopub.execute_input":"2023-01-17T15:51:02.585899Z","iopub.status.idle":"2023-01-17T15:51:02.845997Z","shell.execute_reply.started":"2023-01-17T15:51:02.585850Z","shell.execute_reply":"2023-01-17T15:51:02.844664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:02.847699Z","iopub.execute_input":"2023-01-17T15:51:02.848086Z","iopub.status.idle":"2023-01-17T15:51:02.858320Z","shell.execute_reply.started":"2023-01-17T15:51:02.848053Z","shell.execute_reply":"2023-01-17T15:51:02.857340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas = full_pipeline.predict_proba(test)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:51:02.859643Z","iopub.execute_input":"2023-01-17T15:51:02.860045Z","iopub.status.idle":"2023-01-17T15:54:58.935320Z","shell.execute_reply.started":"2023-01-17T15:51:02.860011Z","shell.execute_reply":"2023-01-17T15:54:58.933453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:54:58.937552Z","iopub.execute_input":"2023-01-17T15:54:58.938157Z","iopub.status.idle":"2023-01-17T15:55:31.201774Z","shell.execute_reply.started":"2023-01-17T15:54:58.938088Z","shell.execute_reply":"2023-01-17T15:55:31.200232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tests = test.groupby('customer_ID').tail(1)\n\ntests.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:31.204145Z","iopub.execute_input":"2023-01-17T15:55:31.204607Z","iopub.status.idle":"2023-01-17T15:55:35.361497Z","shell.execute_reply.started":"2023-01-17T15:55:31.204567Z","shell.execute_reply":"2023-01-17T15:55:35.359938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probas[:,1]","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:35.363187Z","iopub.execute_input":"2023-01-17T15:55:35.363664Z","iopub.status.idle":"2023-01-17T15:55:35.373324Z","shell.execute_reply.started":"2023-01-17T15:55:35.363625Z","shell.execute_reply":"2023-01-17T15:55:35.371844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tests['prediction']=test_probas[:,1]\ntests['prediction']","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:35.375104Z","iopub.execute_input":"2023-01-17T15:55:35.375558Z","iopub.status.idle":"2023-01-17T15:55:35.396709Z","shell.execute_reply.started":"2023-01-17T15:55:35.375515Z","shell.execute_reply":"2023-01-17T15:55:35.394680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tests.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:35.409948Z","iopub.execute_input":"2023-01-17T15:55:35.410433Z","iopub.status.idle":"2023-01-17T15:55:35.420540Z","shell.execute_reply.started":"2023-01-17T15:55:35.410381Z","shell.execute_reply":"2023-01-17T15:55:35.419008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = tests[['customer_ID','prediction']]\nsub.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:35.423073Z","iopub.execute_input":"2023-01-17T15:55:35.423640Z","iopub.status.idle":"2023-01-17T15:55:35.459083Z","shell.execute_reply.started":"2023-01-17T15:55:35.423593Z","shell.execute_reply":"2023-01-17T15:55:35.457409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"my_submission.csv\", index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:35.461028Z","iopub.execute_input":"2023-01-17T15:55:35.461631Z","iopub.status.idle":"2023-01-17T15:55:38.821684Z","shell.execute_reply.started":"2023-01-17T15:55:35.461575Z","shell.execute_reply":"2023-01-17T15:55:38.820286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:38.823208Z","iopub.execute_input":"2023-01-17T15:55:38.823586Z","iopub.status.idle":"2023-01-17T15:55:38.831953Z","shell.execute_reply.started":"2023-01-17T15:55:38.823554Z","shell.execute_reply":"2023-01-17T15:55:38.830304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameter tuning","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import GridSearchCV\n# from sklearn.ensemble import StackingClassifier\n# from sklearn.linear_model import LogisticRegression\n\n# parameters_gb = {\n#   'estimators__lgbm__n_estimators' :[ 2000, 3000, 4000],\n#   'estimators__lgbm__max_depth' : [7,8],\n#   'estimators__lgbm__learning_rate' : [0.01, 0.02],\n#   'estimators__lgbm__bagging_fraction' : [0.6, 0.8],\n#   'estimators__lgbm__feature_fraction' : [0.4, 0.6],\n# #   'estimator__cbc__n_estimators' :[  2000, 3000, 4000],\n# #   'estimator__cbc__max_depth' : [8,9 ],\n# #   'estimator__cbc__learning_rate' : [ 0.02, 0.03],\n# #   'estimator__cbc__subsample' : [0.4, 0.6],\n# #   'estimator__cbc__subsample' : [0.3, 0.5],\n# }\n\n# est_xgb = XGBClassifier()\n# est_cbc = CatBoostClassifier()\n# est_lgbm = LGBMClassifier()\n\n# estimators_st = [\n#     ('lgbm', est_lgbm),          \n#     ('cbc', est_cbc)\n# ]\n\n\n# stacked_estimator =  StackingClassifier(estimators=estimators_st,\n#                                     final_estimator=LogisticRegression,\n#                                     stack_method='predict_proba'\n#                                   )\n\n# full_pipeline_gs = GridSearchCV(estimator =Pipeline([\n#     (\"preprocessor\", preprocessor),\n#     (\"estimators\",   lgbm)\n# ]),  param_grid=parameters_gb)\n\n\n# #full_pipeline_gs.fit(Features, target)\n# # full_pipeline_gs.estimator[1].estimators[0].get_params().keys()\n# full_pipeline_gs.estimator.get_params().keys()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:38.833427Z","iopub.execute_input":"2023-01-17T15:55:38.834189Z","iopub.status.idle":"2023-01-17T15:55:38.844348Z","shell.execute_reply.started":"2023-01-17T15:55:38.834145Z","shell.execute_reply":"2023-01-17T15:55:38.843050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\" Results from Grid Search \")\n# print(\"\\n The best estimator across ALL searched params:\\n\",full_pipeline_gs.best_estimator_)\n# print(\"\\n The best score across ALL searched params:\\n\",full_pipeline_gs.best_score_)\n# print(\"\\n The best parameters across ALL searched params:\\n\",full_pipeline_gs.best_params_)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:55:38.846222Z","iopub.execute_input":"2023-01-17T15:55:38.846680Z","iopub.status.idle":"2023-01-17T15:55:38.858905Z","shell.execute_reply.started":"2023-01-17T15:55:38.846642Z","shell.execute_reply":"2023-01-17T15:55:38.857180Z"},"trusted":true},"execution_count":null,"outputs":[]}]}