{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ensemble Prediction Model\n\nThis notebook shows how to use ensemble for python.\n\nData preprocessing is taken from [AMEX LightGBM Quickstart](https://www.kaggle.com/code/ambrosm/amex-lightgbm-quickstart)\nUses a lot of code from [This Notebook too](https://www.kaggle.com/code/ldfreeman3/a-data-science-framework-to-achieve-99-accuracy)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nfrom colorama import Fore, Back, Style\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.calibration import CalibrationDisplay\nfrom lightgbm import LGBMClassifier, log_evaluation\n\nplt.rcParams['axes.facecolor'] = '#0057b8' # blue\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nplt.rcParams['text.color'] = 'w'\n\nINFERENCE = True # set to False if you only want to cross-validate\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-30T03:11:15.780150Z","iopub.execute_input":"2022-05-30T03:11:15.780709Z","iopub.status.idle":"2022-05-30T03:11:15.794095Z","shell.execute_reply.started":"2022-05-30T03:11:15.780672Z","shell.execute_reply":"2022-05-30T03:11:15.792565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(y_true, y_pred),\n            True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-30T03:11:15.796452Z","iopub.execute_input":"2022-05-30T03:11:15.796973Z","iopub.status.idle":"2022-05-30T03:11:15.817720Z","shell.execute_reply.started":"2022-05-30T03:11:15.796933Z","shell.execute_reply":"2022-05-30T03:11:15.816832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we create two groups of features:\n- Selected features averaged over all statements of a customer\n- Selected features taken from the last statement of a customer\n\nThe code has been optimized for memory efficiency rather than readability. In particular, `.iloc[mask_array, columns]` needs much less RAM than the groupby construction used in the previous version of the notebook.\n","metadata":{}},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_1', 'B_11', 'B_16', 'B_17', 'B_18', 'B_2', 'B_20',\n                'B_28', 'B_3', 'B_4', 'B_5', 'B_7', 'B_9', 'D_112',\n                'D_121', 'D_141', 'D_39', 'D_41', 'D_42', 'D_43',\n                'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', \n                'D_50', 'D_51', 'D_53', 'D_54', 'D_56', 'D_58', \n                'D_59', 'D_60', 'D_91', 'P_2', 'P_3', 'R_1', 'R_2', \n                'R_27', 'R_3', 'R_7', 'S_11', 'S_26', 'S_3', 'S_5']\nfeatures_last = ['B_1', 'B_10', 'B_11', 'B_12', 'B_13', 'B_15', 'B_16',\n                 'B_17', 'B_18', 'B_19', 'B_2', 'B_20', 'B_22', 'B_23',\n                 'B_24', 'B_25', 'B_26', 'B_27', 'B_28', 'B_29', 'B_3',\n                 'B_32', 'B_33', 'B_36', 'B_38', 'B_39', 'B_4', 'B_40',\n                 'B_41', 'B_42', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9',\n                 'D_102', 'D_103', 'D_105', 'D_106', 'D_107', 'D_109',\n                 'D_112', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120',\n                 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_127', \n                 'D_129', 'D_132', 'D_133', 'D_135', 'D_136', 'D_137', \n                 'D_140', 'D_141', 'D_143', 'D_145', 'D_39', 'D_41',\n                 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48',\n                 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55',\n                 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63',\n                 'D_64', 'D_66', 'D_70', 'D_72', 'D_73', 'D_74', 'D_75',\n                 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_82', 'D_83',\n                 'D_84', 'D_86', 'D_91', 'D_92', 'D_93', 'D_94', 'D_96',\n                 'P_2', 'P_3', 'R_1', 'R_10', 'R_11', 'R_12', 'R_13',\n                 'R_14', 'R_15', 'R_17', 'R_18', 'R_19', 'R_2', 'R_20', \n                 'R_21', 'R_22', 'R_24', 'R_25', 'R_26', 'R_27', 'R_3',\n                 'R_4', 'R_5', 'R_7', 'R_8', 'R_9', 'S_11', 'S_12',\n                 'S_13', 'S_15', 'S_17', 'S_20', 'S_22', 'S_23', \n                 'S_24', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_6',\n                 'S_7', 'S_8', 'S_9']\n\ntrain_test = [None, None] # first element is train, second element is test\nfor i in [1, 0] if INFERENCE else [0]:\n    train_test[i] = pd.read_feather(['../input/amexfeather/train_data.ftr',\n                                     '../input/amexfeather/test_data.ftr'][i])\n    cid = pd.Categorical(train_test[i].pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if i == 0: # train\n        target = train_test[0].loc[last, 'target']\n    gc.collect()\n    print('Read', i)\n    df_avg = (train_test[i][features_avg]\n              .groupby(cid)\n              .mean()\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    train_test[i] = (train_test[i].loc[last, features_last]\n                     .rename(columns={f: f\"{f}_last\" for f in features_last})\n                     .set_index(np.asarray(cid[last]))\n                    )\n    gc.collect()\n    print('Computed last', i)\n    train_test[i] = pd.concat([train_test[i], df_avg], axis=1)\n    del df_avg, cid, last\n\ntrain, test = tuple(train_test)\ndel train_test\nif INFERENCE: print('Shapes:', train.shape, target.shape, test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:11:15.819640Z","iopub.execute_input":"2022-05-30T03:11:15.820284Z","iopub.status.idle":"2022-05-30T03:12:51.852371Z","shell.execute_reply.started":"2022-05-30T03:11:15.820237Z","shell.execute_reply":"2022-05-30T03:12:51.850681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing\n\n","metadata":{}},{"cell_type":"code","source":"train_numericcols = train.select_dtypes('float16').columns.tolist()\ntrain_catcols = train.select_dtypes(exclude=['float16','int64']).columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:12:51.854409Z","iopub.execute_input":"2022-05-30T03:12:51.854907Z","iopub.status.idle":"2022-05-30T03:12:52.573956Z","shell.execute_reply.started":"2022-05-30T03:12:51.854866Z","shell.execute_reply":"2022-05-30T03:12:52.572538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation\nNeed to understand the correlation between the numerical variables and drop ones with correlation greater than 0.9","metadata":{"execution":{"iopub.status.busy":"2022-05-28T10:25:04.531875Z","iopub.execute_input":"2022-05-28T10:25:04.532287Z","iopub.status.idle":"2022-05-28T10:25:04.539915Z","shell.execute_reply.started":"2022-05-28T10:25:04.532256Z","shell.execute_reply":"2022-05-28T10:25:04.539102Z"}}},{"cell_type":"code","source":"import numpy as np\n\n# Create correlation matrix\ncorr_matrix = train[train_numericcols].corr().abs()\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n# Find features with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] > 0.9)]\n\n# Drop features \ntrain.drop(to_drop, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:12:52.576859Z","iopub.execute_input":"2022-05-30T03:12:52.577336Z","iopub.status.idle":"2022-05-30T03:13:39.072915Z","shell.execute_reply.started":"2022-05-30T03:12:52.577302Z","shell.execute_reply":"2022-05-30T03:13:39.071497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(str(len(to_drop)) + ' Columns were dropped')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:39.074904Z","iopub.execute_input":"2022-05-30T03:13:39.075469Z","iopub.status.idle":"2022-05-30T03:13:39.083015Z","shell.execute_reply.started":"2022-05-30T03:13:39.075417Z","shell.execute_reply":"2022-05-30T03:13:39.081636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'] = target.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:39.084919Z","iopub.execute_input":"2022-05-30T03:13:39.085407Z","iopub.status.idle":"2022-05-30T03:13:39.306631Z","shell.execute_reply.started":"2022-05-30T03:13:39.085360Z","shell.execute_reply":"2022-05-30T03:13:39.305311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Missing Values\n\nWhile some algos deal with missing values automatically, most don't. So for missing values we will be following a simple philosophy:\n\n* Numeric Columns: Replace with Median\n* Categorical columns: Take WOE, this automatically treats missing as a category\n\n## Numeric","metadata":{}},{"cell_type":"code","source":"# remove correlated variables\ntrain_numericcols2 = set(train_numericcols) - set(to_drop)\n\n#replace with median \n\nfor col in train_numericcols2:    \n    train[col].fillna(train[col].median(), inplace = True)\n    test[col].fillna(test[col].median(), inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:39.308815Z","iopub.execute_input":"2022-05-30T03:13:39.309237Z","iopub.status.idle":"2022-05-30T03:13:48.310582Z","shell.execute_reply.started":"2022-05-30T03:13:39.309205Z","shell.execute_reply":"2022-05-30T03:13:48.309357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical","metadata":{}},{"cell_type":"code","source":"# WOE transformation of categorical variables \n\ndef get_bivar_tables(train_data, col_names, dep_var):\n    for col_name in col_names:\n            print(col_name)\n            train_data[col_name] = train_data[col_name].cat.add_categories(\"Missing\").fillna(\"Missing\")\n            train_data['bins'] = train_data[col_name].replace(np.nan,'Missing').replace('','Missing')\n            binned_df = train_data.groupby(['bins'], as_index=False)[dep_var].count()\n            binned_df['events'] = train_data.groupby(['bins'])[dep_var].sum().tolist()\n            binned_df['non_events'] = binned_df[dep_var] - binned_df['events']\n            binned_df['event_rate'] = [x*100 for x in train_data.groupby(['bins'])[dep_var].mean().tolist()]\n            binned_df['event_rate'] = binned_df['event_rate'].replace(np.nan,0)\n            binned_df['event_capture'] = np.round((binned_df['events']/np.sum(binned_df['events'])),8)\n            binned_df['non_event_capture'] = np.round((binned_df['non_events']/np.sum(binned_df['non_events'])),8)\n            binned_df['woe'] = np.log(binned_df['non_event_capture']/binned_df['event_capture']).replace(np.nan,0).replace(np.inf,0)\n            binned_df['IV'] = ((binned_df['non_event_capture'] - binned_df['event_capture'])*binned_df['woe']).replace(np.nan,0).replace(np.inf,0)\n            bivar_tables[col_name] = binned_df\n           # print(col_name + \":\" + \"\\n\")\n           # print(info_val[col_name])\n    return bivar_tables\n\nbivar_tables = {}\nbivar_tables= get_bivar_tables(train, train_catcols, 'target')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:48.312351Z","iopub.execute_input":"2022-05-30T03:13:48.312678Z","iopub.status.idle":"2022-05-30T03:13:48.478775Z","shell.execute_reply.started":"2022-05-30T03:13:48.312650Z","shell.execute_reply":"2022-05-30T03:13:48.476970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_woe(train_data, val_data, bivar_tables,vars_used2):\n    for col_name in vars_used2:\n        print(col_name)\n        val_data[col_name] = val_data[col_name].cat.add_categories(\"Missing\").fillna(\"Missing\")\n        \n        train_data['bins'] = train_data[col_name].replace(np.nan,'Missing').replace('','Missing')\n        val_data['bins'] = val_data[col_name].replace(np.nan,'Missing').replace('','Missing')\n      \n\n        train_data[col_name]= list(pd.merge(train_data[[col_name,'bins']], bivar_tables[col_name][['bins','woe']], on=['bins'], how='left')['woe'])\n        val_data[col_name]= list(pd.merge(val_data[[col_name,'bins']], bivar_tables[col_name][['bins','woe']], on=['bins'], how='left')['woe'])\n\n\n    return train_data,val_data \n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:48.482749Z","iopub.execute_input":"2022-05-30T03:13:48.483141Z","iopub.status.idle":"2022-05-30T03:13:48.493906Z","shell.execute_reply.started":"2022-05-30T03:13:48.483109Z","shell.execute_reply":"2022-05-30T03:13:48.492439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create final datasets","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:51:08.15137Z","iopub.execute_input":"2022-05-29T07:51:08.152481Z","iopub.status.idle":"2022-05-29T07:51:08.182048Z","shell.execute_reply.started":"2022-05-29T07:51:08.152422Z","shell.execute_reply":"2022-05-29T07:51:08.180869Z"}}},{"cell_type":"code","source":"train_df1 = train.copy()\nval_df1 = test.copy()\ntrain_df1, val_df1 = get_woe(train_df1, val_df1,bivar_tables, train_catcols)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:48.496267Z","iopub.execute_input":"2022-05-30T03:13:48.496724Z","iopub.status.idle":"2022-05-30T03:13:53.049451Z","shell.execute_reply.started":"2022-05-30T03:13:48.496688Z","shell.execute_reply":"2022-05-30T03:13:53.048121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finalcols\n\nfinal_cols = list(train_numericcols2) + train_catcols","metadata":{"execution":{"iopub.status.busy":"2022-05-30T03:13:53.051026Z","iopub.execute_input":"2022-05-30T03:13:53.051390Z","iopub.status.idle":"2022-05-30T03:13:53.057721Z","shell.execute_reply.started":"2022-05-30T03:13:53.051360Z","shell.execute_reply":"2022-05-30T03:13:53.056320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Train and Test Data\n\nAs mentioned previously, the test file provided is really validation data for competition submission. So, we will use sklearn function to split the training data in two datasets; 75/25 split. This is important, so we don't overfit our model. Meaning, the algorithm is so specific to a given subset, it cannot accurately generalize another subset, from the same dataset. It's important our algorithm has not seen the subset we will use to test, so it doesn't \"cheat\" by memorizing the answers. We will use sklearn's train_test_split function. In later sections we will also use sklearn's cross validation functions, that splits our dataset into train and test for data modeling comparison.","metadata":{}},{"cell_type":"code","source":"from sklearn import model_selection  \n\ntrain_1, val_1 = model_selection.train_test_split(train_df1, random_state = 0)\n\nprint(\"Train1 Shape: {}\".format(train_1.shape))\nprint(\"Test1 Shape: {}\".format(val_1.shape))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:18:49.55207Z","iopub.execute_input":"2022-05-29T09:18:49.552517Z","iopub.status.idle":"2022-05-29T09:18:50.879558Z","shell.execute_reply.started":"2022-05-29T09:18:49.552477Z","shell.execute_reply":"2022-05-29T09:18:50.878527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n## Define Models","metadata":{}},{"cell_type":"code","source":"from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process\nfrom xgboost import XGBClassifier\n\n#Common Model Helpers\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn import feature_selection\nfrom sklearn import model_selection\nfrom sklearn import metrics\n\nvote_est = [\n    #Ensemble Methods: http://scikit-learn.org/stable/modules/ensemble.html\n    ('ada', ensemble.AdaBoostClassifier()),\n  \n    ('etc',ensemble.ExtraTreesClassifier()),\n    ('rfc', ensemble.RandomForestClassifier()),\n\n\n    \n    #GLM: http://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n    ('lr', linear_model.LogisticRegressionCV()),\n    \n    #Navies Bayes: http://scikit-learn.org/stable/modules/naive_bayes.html\n    ('bnb', naive_bayes.BernoulliNB()),\n    ('gnb', naive_bayes.GaussianNB()),\n\n    \n    #xgboost: http://xgboost.readthedocs.io/en/latest/model.html\n   ('xgb', XGBClassifier())\n\n]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T08:22:29.51572Z","iopub.execute_input":"2022-05-29T08:22:29.516142Z","iopub.status.idle":"2022-05-29T08:22:29.882884Z","shell.execute_reply.started":"2022-05-29T08:22:29.51611Z","shell.execute_reply":"2022-05-29T08:22:29.881556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining Hyperparameters","metadata":{}},{"cell_type":"code","source":"#Hyperparameter Tune with GridSearchCV: http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html\ngrid_n_estimator = [10, 50, 100, 300]\ngrid_ratio = [.1, .25, .5, .75, 1.0]\ngrid_learn = [.01, .03, .05, .1, .25]\ngrid_max_depth = [2, 4, 6, 8, 10]\ngrid_min_samples = [5, 10, .03, .05, .10]\ngrid_criterion = ['gini', 'entropy']\ngrid_bool = [True, False]\ngrid_seed = [0]\n\n\ngrid_param = [\n            {\n            #AdaBoostClassifier - http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html\n            'n_estimators': grid_n_estimator, #default=50\n            'learning_rate': grid_learn, #default=1\n            #'algorithm': ['SAMME', 'SAMME.R'], #default=’SAMME.R\n            'random_state': grid_seed\n            },\n       \n    \n\n    \n            {\n            #ExtraTreesClassifier - http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier\n            'n_estimators': grid_n_estimator, #default=10\n            'criterion': grid_criterion, #default=”gini”\n            'max_depth': grid_max_depth, #default=None\n            'random_state': grid_seed\n             },\n\n\n    \n            {\n            #RandomForestClassifier - http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier\n            'n_estimators': grid_n_estimator, #default=10\n            'criterion': grid_criterion, #default=”gini”\n            'max_depth': grid_max_depth, #default=None\n            'oob_score': [True], #default=False -- 12/31/17 set to reduce runtime -- The best parameter for RandomForestClassifier is {'criterion': 'entropy', 'max_depth': 6, 'n_estimators': 100, 'oob_score': True, 'random_state': 0} with a runtime of 146.35 seconds.\n            'random_state': grid_seed\n             },\n    \n     \n    \n            {\n            #LogisticRegressionCV - http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegressionCV.html#sklearn.linear_model.LogisticRegressionCV\n            'fit_intercept': grid_bool, #default: True\n            #'penalty': ['l1','l2'],\n            'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'], #default: lbfgs\n            'random_state': grid_seed\n             },\n            \n    \n            {\n            #BernoulliNB - http://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html#sklearn.naive_bayes.BernoulliNB\n            'alpha': grid_ratio, #default: 1.0\n             },\n    \n     #GaussianNB - \n            {},\n\n    \n            {\n            #XGBClassifier - http://xgboost.readthedocs.io/en/latest/parameter.html\n            'learning_rate': grid_learn, #default: .3\n            'max_depth': [1,2,4,6,8,10], #default 2\n            'n_estimators': grid_n_estimator, \n            'seed': grid_seed  \n             }   \n        ]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T08:35:24.202673Z","iopub.execute_input":"2022-05-29T08:35:24.203209Z","iopub.status.idle":"2022-05-29T08:35:24.220845Z","shell.execute_reply.started":"2022-05-29T08:35:24.203163Z","shell.execute_reply":"2022-05-29T08:35:24.220118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tune Params\n\nFor tuning, we can use a sample of variables, if you have got resources you can run on entire train set","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV\n\nparam_comb = 25\n\n# sample 10000 for tuning\n\ntrain_samp = train_1.sample(10000)\nfor clf, param in zip (vote_est, grid_param):\n    best_search = RandomizedSearchCV(estimator = clf[1], param_distributions = param, n_iter=param_comb,cv = 5, scoring = 'roc_auc', verbose=3, n_jobs = 125, random_state=1001)\n    best_search.fit(train_samp[final_cols],train_samp['target'])\n    best_param = best_search.best_params_\n\n    print('The best parameter for {} is {}'.format(clf[1].__class__.__name__, best_param)) \n    clf[1].set_params(**best_param)\n    print('-'*10)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T08:47:08.755537Z","iopub.execute_input":"2022-05-29T08:47:08.756102Z","iopub.status.idle":"2022-05-29T09:10:45.445505Z","shell.execute_reply.started":"2022-05-29T08:47:08.756052Z","shell.execute_reply":"2022-05-29T09:10:45.442916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fit the tuned models\nFor now, we are just fitting on the train set, in next iteration we will do statistics on train and test to check for overfitting","metadata":{}},{"cell_type":"code","source":"# For now, \n\nvote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft',n_jobs = -1)\nvote_soft.fit(train_df1[final_cols],train_df1['target'])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:10:45.447578Z","iopub.status.idle":"2022-05-29T09:10:45.466588Z","shell.execute_reply.started":"2022-05-29T09:10:45.466063Z","shell.execute_reply":"2022-05-29T09:10:45.466127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = vote_soft.predict_proba(val_df1[final_cols])[:, 1]","metadata":{"execution":{"iopub.status.busy":"2022-05-29T09:10:45.503362Z","iopub.status.idle":"2022-05-29T09:10:45.507338Z","shell.execute_reply.started":"2022-05-29T09:10:45.506979Z","shell.execute_reply":"2022-05-29T09:10:45.507018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\n","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame({'customer_ID': test.index,\n                        'prediction': y_pred})\nsub.to_csv('submission.csv', index=False)\ndisplay(sub)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T08:05:33.825821Z","iopub.status.idle":"2022-05-29T08:05:33.826177Z","shell.execute_reply.started":"2022-05-29T08:05:33.826007Z","shell.execute_reply":"2022-05-29T08:05:33.826025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}