{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.040763,"end_time":"2022-08-03T14:46:45.678879","exception":false,"start_time":"2022-08-03T14:46:45.638116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:12:59.473538Z","iopub.execute_input":"2022-08-30T09:12:59.474210Z","iopub.status.idle":"2022-08-30T09:12:59.518231Z","shell.execute_reply.started":"2022-08-30T09:12:59.474079Z","shell.execute_reply":"2022-08-30T09:12:59.516866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings,gc\nwarnings.filterwarnings('ignore')\n\ndf = pd.read_feather('../input/amexfeather/train_data.ftr')\ndf_train1 =  (df\n            .groupby('customer_ID')\n            .tail(1))\n\ndf_train2 = (df\n            .groupby('customer_ID')\n            .tail(2))\ndf_train1.S_2 = 3\ndf_train2.S_2 = 2\ndf_train = pd.concat([df_train1,df_train2], ignore_index=True)\ndf_train","metadata":{"papermill":{"duration":31.208234,"end_time":"2022-08-03T14:47:16.900270","exception":false,"start_time":"2022-08-03T14:46:45.692036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:13:02.473114Z","iopub.execute_input":"2022-08-30T09:13:02.473667Z","iopub.status.idle":"2022-08-30T09:13:29.329622Z","shell.execute_reply.started":"2022-08-30T09:13:02.473625Z","shell.execute_reply":"2022-08-30T09:13:29.328542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"papermill":{"duration":0.150641,"end_time":"2022-08-03T14:47:17.064873","exception":false,"start_time":"2022-08-03T14:47:16.914232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:13:44.532616Z","iopub.execute_input":"2022-08-30T09:13:44.533812Z","iopub.status.idle":"2022-08-30T09:13:44.680483Z","shell.execute_reply.started":"2022-08-30T09:13:44.533769Z","shell.execute_reply":"2022-08-30T09:13:44.679372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### For feature selection we have picked top 85 numerical features with about 99% cumulative importance.Among categorical features we have picked all but D_68 for now.\n\nhttps://www.kaggle.com/code/vijayendrad/model-5-amex-feature-importances","metadata":{"papermill":{"duration":0.01266,"end_time":"2022-08-03T14:47:17.090535","exception":false,"start_time":"2022-08-03T14:47:17.077875","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#features selected using the notebook above\nall_f = ['customer_ID','S_2','B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64','B_1','B_10','B_11',\n         'B_12','B_13','B_14','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_26','B_3','B_33','B_36','B_37',\n         'B_4','B_40','B_5','B_6','B_7','B_8','B_9','D_112','D_113','D_115','D_118','D_119','D_121','D_122','D_128','D_129',\n         'D_131','D_133','D_39','D_41','D_44','D_45','D_47','D_48','D_51','D_52','D_54','D_55','D_58','D_59','D_61','D_62',\n         'D_65','D_71','D_74','D_75','D_78','D_79','D_81','D_84','P_2','P_3','P_4','R_1','R_10','R_11','R_16','R_2','R_27',\n         'R_3','R_4','R_5','R_6','R_7','S_13','S_15','S_16','S_22','S_23','S_24','S_25','S_26','S_3','S_7','S_8','target']","metadata":{"papermill":{"duration":0.024328,"end_time":"2022-08-03T14:47:17.127803","exception":false,"start_time":"2022-08-03T14:47:17.103475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:13:47.445814Z","iopub.execute_input":"2022-08-30T09:13:47.446238Z","iopub.status.idle":"2022-08-30T09:13:47.456002Z","shell.execute_reply.started":"2022-08-30T09:13:47.446204Z","shell.execute_reply":"2022-08-30T09:13:47.453998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_f = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64'] \n\nunused = ['D_68']\nall_f.remove(\"customer_ID\")\n#all_f.remove(\"S_2\")\nall_f.remove(\"target\")\n\nnum_f = list(set(all_f) - set(cat_f) - set(unused))\n\ndfnum = df_train[num_f]\ndfcat = df_train[cat_f]\n\ntest = dfnum.dropna()\n\nfor col in num_f:\n    dfnum[col] = dfnum[col].fillna(test[col].median())","metadata":{"papermill":{"duration":5.849613,"end_time":"2022-08-03T14:47:22.989888","exception":false,"start_time":"2022-08-03T14:47:17.140275","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:13:54.962847Z","iopub.execute_input":"2022-08-30T09:13:54.963298Z","iopub.status.idle":"2022-08-30T09:14:00.226790Z","shell.execute_reply.started":"2022-08-30T09:13:54.963262Z","shell.execute_reply":"2022-08-30T09:14:00.225648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_feather('/kaggle/input/amexfeather/test_data.ftr')\n\ndf_test =  (test_df\n            .groupby('customer_ID')\n            .tail(1))\nall_f.append(\"customer_ID\")\n\ndfn_test = df_test[all_f]\n\n\n\ndel test_df,df_test\ngc.collect()","metadata":{"papermill":{"duration":53.613243,"end_time":"2022-08-03T14:48:16.617978","exception":false,"start_time":"2022-08-03T14:47:23.004735","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:14:04.052207Z","iopub.execute_input":"2022-08-30T09:14:04.053100Z","iopub.status.idle":"2022-08-30T09:14:46.187177Z","shell.execute_reply.started":"2022-08-30T09:14:04.053049Z","shell.execute_reply":"2022-08-30T09:14:46.186065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfn_test[\"S_2\"] = pd.to_datetime(dfn_test[\"S_2\"])\ndfn_test[\"S_2day\"] = dfn_test[\"S_2\"].dt.weekday\ndfn_test[\"S_2month\"] = dfn_test[\"S_2\"].dt.month\ndfn_test[\"S_2year\"] = dfn_test[\"S_2\"].dt.year","metadata":{"execution":{"iopub.status.busy":"2022-08-30T09:17:24.619944Z","iopub.execute_input":"2022-08-30T09:17:24.620498Z","iopub.status.idle":"2022-08-30T09:17:24.961523Z","shell.execute_reply.started":"2022-08-30T09:17:24.620456Z","shell.execute_reply":"2022-08-30T09:17:24.960296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfn_test[\"S_2month\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T09:26:10.944652Z","iopub.execute_input":"2022-08-30T09:26:10.945139Z","iopub.status.idle":"2022-08-30T09:26:10.962661Z","shell.execute_reply.started":"2022-08-30T09:26:10.945098Z","shell.execute_reply":"2022-08-30T09:26:10.961381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfn_test[\"S_2month\"] = dfn_test[\"S_2month\"].replace(to_replace=10, value= 22)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-30T09:24:17.772330Z","iopub.execute_input":"2022-08-30T09:24:17.773389Z","iopub.status.idle":"2022-08-30T09:24:17.800841Z","shell.execute_reply.started":"2022-08-30T09:24:17.773324Z","shell.execute_reply":"2022-08-30T09:24:17.799777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfn_test.S_2 = dfn_test.S_2month","metadata":{"execution":{"iopub.status.busy":"2022-08-30T09:26:37.131816Z","iopub.execute_input":"2022-08-30T09:26:37.132412Z","iopub.status.idle":"2022-08-30T09:26:37.144862Z","shell.execute_reply.started":"2022-08-30T09:26:37.132368Z","shell.execute_reply":"2022-08-30T09:26:37.143365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfn_test = dfn_test.drop(['S_2day','S_2month','S_2year'],axis=1)\ndfn_test","metadata":{"execution":{"iopub.status.busy":"2022-08-30T09:27:58.486215Z","iopub.execute_input":"2022-08-30T09:27:58.486770Z","iopub.status.idle":"2022-08-30T09:27:59.099612Z","shell.execute_reply.started":"2022-08-30T09:27:58.486727Z","shell.execute_reply":"2022-08-30T09:27:59.098395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sn\nfor col in cat_f:\n    \n    plt.figure(figsize=(7,5))\n\n    ax = sn.countplot(x=col, data=dfcat)\n    for p in ax.patches:#displaying % as annotations\n        ax.annotate(str(round(100*p.get_height()/len(dfcat),2))+\"%\", (p.get_x()+0.2, p.get_height()+2))","metadata":{"papermill":{"duration":3.698647,"end_time":"2022-08-03T14:48:20.329719","exception":false,"start_time":"2022-08-03T14:48:16.631072","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:21.046406Z","iopub.execute_input":"2022-08-30T09:28:21.046955Z","iopub.status.idle":"2022-08-30T09:28:23.875375Z","shell.execute_reply.started":"2022-08-30T09:28:21.046901Z","shell.execute_reply":"2022-08-30T09:28:23.874122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = \"target\"\n\nint_columns = num_f\n\nfeature_columns = (\n    int_columns + cat_f + [target])","metadata":{"papermill":{"duration":0.025989,"end_time":"2022-08-03T14:48:20.371671","exception":false,"start_time":"2022-08-03T14:48:20.345682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:29.338658Z","iopub.execute_input":"2022-08-30T09:28:29.339987Z","iopub.status.idle":"2022-08-30T09:28:29.347045Z","shell.execute_reply.started":"2022-08-30T09:28:29.339872Z","shell.execute_reply":"2022-08-30T09:28:29.345406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfcat['D_64'] = dfcat['D_64'].replace([\"\"],\"unknown\")","metadata":{"papermill":{"duration":0.029817,"end_time":"2022-08-03T14:48:20.416859","exception":false,"start_time":"2022-08-03T14:48:20.387042","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:32.484972Z","iopub.execute_input":"2022-08-30T09:28:32.485472Z","iopub.status.idle":"2022-08-30T09:28:32.495944Z","shell.execute_reply.started":"2022-08-30T09:28:32.485434Z","shell.execute_reply":"2022-08-30T09:28:32.495008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create two DataFrames, one for each data type\n\ntraindata_num = dfnum\ntraindata_cat = dfcat","metadata":{"papermill":{"duration":0.025034,"end_time":"2022-08-03T14:48:20.458323","exception":false,"start_time":"2022-08-03T14:48:20.433289","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:35.885368Z","iopub.execute_input":"2022-08-30T09:28:35.885836Z","iopub.status.idle":"2022-08-30T09:28:35.891625Z","shell.execute_reply.started":"2022-08-30T09:28:35.885797Z","shell.execute_reply":"2022-08-30T09:28:35.890546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(traindata_num.isnull().sum())\nprint(traindata_cat.isnull().sum())","metadata":{"papermill":{"duration":0.643325,"end_time":"2022-08-03T14:48:21.119112","exception":false,"start_time":"2022-08-03T14:48:20.475787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:38.615061Z","iopub.execute_input":"2022-08-30T09:28:38.615746Z","iopub.status.idle":"2022-08-30T09:28:39.200263Z","shell.execute_reply.started":"2022-08-30T09:28:38.615707Z","shell.execute_reply":"2022-08-30T09:28:39.198975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor col in cat_f:\n    traindata_cat[col] = traindata_cat[col].fillna(traindata_cat[col].value_counts().idxmax())   \n    \nprint(traindata_cat.isnull().sum())\n\n","metadata":{"papermill":{"duration":0.16088,"end_time":"2022-08-03T14:48:21.296075","exception":false,"start_time":"2022-08-03T14:48:21.135195","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:42.586588Z","iopub.execute_input":"2022-08-30T09:28:42.587033Z","iopub.status.idle":"2022-08-30T09:28:42.736165Z","shell.execute_reply.started":"2022-08-30T09:28:42.586994Z","shell.execute_reply":"2022-08-30T09:28:42.734647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputed_train = pd.concat([traindata_cat, traindata_num], axis=1, join='inner')\nimputed_train\n\n","metadata":{"papermill":{"duration":0.810696,"end_time":"2022-08-03T14:48:22.122904","exception":false,"start_time":"2022-08-03T14:48:21.312208","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:47.464664Z","iopub.execute_input":"2022-08-30T09:28:47.465098Z","iopub.status.idle":"2022-08-30T09:28:48.108585Z","shell.execute_reply.started":"2022-08-30T09:28:47.465063Z","shell.execute_reply":"2022-08-30T09:28:48.107318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputed_train['target'] = df_train.target","metadata":{"papermill":{"duration":0.03722,"end_time":"2022-08-03T14:48:22.177187","exception":false,"start_time":"2022-08-03T14:48:22.139967","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:51.247071Z","iopub.execute_input":"2022-08-30T09:28:51.247501Z","iopub.status.idle":"2022-08-30T09:28:51.258077Z","shell.execute_reply.started":"2022-08-30T09:28:51.247464Z","shell.execute_reply":"2022-08-30T09:28:51.256550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Importing resample from *sklearn.utils* package.\nfrom sklearn.utils import resample\n\n# Separate the case of yes-subscribes and no-subscribes\ndf_0 = imputed_train[imputed_train.target == 0]\ndf_1 = imputed_train[imputed_train.target == 1]\n\n##Upsample the yes-subscribed cases.\ndf_minority_upsampled = resample(df_1, \n                                 replace=True,     # sample with replacement\n                                 n_samples=1000000) \n\n# Combine majority class with upsampled minority class\nnew_df = pd.concat([df_0, df_minority_upsampled])\n\n\nfrom sklearn.utils import shuffle\nnew_df = shuffle(new_df)\nnew_df\n\n","metadata":{"papermill":{"duration":7.635442,"end_time":"2022-08-03T14:48:29.829890","exception":false,"start_time":"2022-08-03T14:48:22.194448","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:28:54.699722Z","iopub.execute_input":"2022-08-30T09:28:54.700187Z","iopub.status.idle":"2022-08-30T09:29:01.365398Z","shell.execute_reply.started":"2022-08-30T09:28:54.700148Z","shell.execute_reply":"2022-08-30T09:29:01.364044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sn.countplot(x=target, data=new_df)\nfor p in ax.patches:\n    \n    #displaying % as annotations\n    ax.annotate(str(round(100*p.get_height()/len(new_df),2))+\"%\", (p.get_x()+0.2, p.get_height()+2))","metadata":{"papermill":{"duration":0.607203,"end_time":"2022-08-03T14:48:30.455253","exception":false,"start_time":"2022-08-03T14:48:29.848050","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:04.988896Z","iopub.execute_input":"2022-08-30T09:29:04.989396Z","iopub.status.idle":"2022-08-30T09:29:05.487291Z","shell.execute_reply.started":"2022-08-30T09:29:04.989356Z","shell.execute_reply":"2022-08-30T09:29:05.486382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df = pd.get_dummies( new_df, \n                                        columns = cat_f,\n                                        drop_first = True )\n\nencoded_df","metadata":{"papermill":{"duration":2.485082,"end_time":"2022-08-03T14:48:32.959125","exception":false,"start_time":"2022-08-03T14:48:30.474043","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:09.675391Z","iopub.execute_input":"2022-08-30T09:29:09.676154Z","iopub.status.idle":"2022-08-30T09:29:11.695029Z","shell.execute_reply.started":"2022-08-30T09:29:09.676112Z","shell.execute_reply":"2022-08-30T09:29:11.693847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df.target","metadata":{"papermill":{"duration":0.032623,"end_time":"2022-08-03T14:48:33.011285","exception":false,"start_time":"2022-08-03T14:48:32.978662","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:17.109789Z","iopub.execute_input":"2022-08-30T09:29:17.110243Z","iopub.status.idle":"2022-08-30T09:29:17.120652Z","shell.execute_reply.started":"2022-08-30T09:29:17.110204Z","shell.execute_reply":"2022-08-30T09:29:17.119445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df = encoded_df.drop(['D_64_-1'],axis=1)","metadata":{"papermill":{"duration":1.088121,"end_time":"2022-08-03T14:48:34.117804","exception":false,"start_time":"2022-08-03T14:48:33.029683","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:21.340855Z","iopub.execute_input":"2022-08-30T09:29:21.341292Z","iopub.status.idle":"2022-08-30T09:29:22.440522Z","shell.execute_reply.started":"2022-08-30T09:29:21.341258Z","shell.execute_reply":"2022-08-30T09:29:22.439246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = encoded_df","metadata":{"papermill":{"duration":0.026953,"end_time":"2022-08-03T14:48:34.163350","exception":false,"start_time":"2022-08-03T14:48:34.136397","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:24.197889Z","iopub.execute_input":"2022-08-30T09:29:24.198350Z","iopub.status.idle":"2022-08-30T09:29:24.204900Z","shell.execute_reply.started":"2022-08-30T09:29:24.198315Z","shell.execute_reply":"2022-08-30T09:29:24.203607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = encoded_df.target\n\nencoded_df = encoded_df.drop(\"target\",axis=1)\n\nX = encoded_df","metadata":{"papermill":{"duration":0.831927,"end_time":"2022-08-03T14:48:35.013198","exception":false,"start_time":"2022-08-03T14:48:34.181271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:29.309422Z","iopub.execute_input":"2022-08-30T09:29:29.309975Z","iopub.status.idle":"2022-08-30T09:29:30.078843Z","shell.execute_reply.started":"2022-08-30T09:29:29.309923Z","shell.execute_reply":"2022-08-30T09:29:30.077495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n\nX_train, X_test, y_train, y_test = train_test_split(X, \n                                                    y, \n                                                    test_size = 0.3, \n                                                    random_state = 42)","metadata":{"papermill":{"duration":4.902055,"end_time":"2022-08-03T14:48:39.935511","exception":false,"start_time":"2022-08-03T14:48:35.033456","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:32.620978Z","iopub.execute_input":"2022-08-30T09:29:32.621424Z","iopub.status.idle":"2022-08-30T09:29:36.533118Z","shell.execute_reply.started":"2022-08-30T09:29:32.621382Z","shell.execute_reply":"2022-08-30T09:29:36.531973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statsmodels.api as sm\n\nlogit = sm.Logit(y_train, X_train)\nlogit_model = logit.fit()","metadata":{"papermill":{"duration":68.718053,"end_time":"2022-08-03T14:49:48.672045","exception":false,"start_time":"2022-08-03T14:48:39.953992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:29:42.757686Z","iopub.execute_input":"2022-08-30T09:29:42.758185Z","iopub.status.idle":"2022-08-30T09:30:40.488485Z","shell.execute_reply.started":"2022-08-30T09:29:42.758149Z","shell.execute_reply":"2022-08-30T09:30:40.486595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logit_model.summary2()","metadata":{"papermill":{"duration":3.896601,"end_time":"2022-08-03T14:49:52.616676","exception":false,"start_time":"2022-08-03T14:49:48.720075","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:30:57.763536Z","iopub.execute_input":"2022-08-30T09:30:57.764044Z","iopub.status.idle":"2022-08-30T09:31:01.473929Z","shell.execute_reply.started":"2022-08-30T09:30:57.763995Z","shell.execute_reply":"2022-08-30T09:31:01.472913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_significant_vars( lm ):\n    var_p_vals_df = pd.DataFrame( lm.pvalues )\n    var_p_vals_df['vars'] = var_p_vals_df.index\n    var_p_vals_df.columns = ['pvals', 'vars']\n    return list( var_p_vals_df[var_p_vals_df.pvals <= 0.05]['vars'] )","metadata":{"papermill":{"duration":0.030591,"end_time":"2022-08-03T14:49:52.666797","exception":false,"start_time":"2022-08-03T14:49:52.636206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:31:22.561184Z","iopub.execute_input":"2022-08-30T09:31:22.561619Z","iopub.status.idle":"2022-08-30T09:31:22.568678Z","shell.execute_reply.started":"2022-08-30T09:31:22.561585Z","shell.execute_reply":"2022-08-30T09:31:22.567493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"significant_vars = get_significant_vars( logit_model )\n\nsignificant_vars","metadata":{"papermill":{"duration":0.038695,"end_time":"2022-08-03T14:49:52.725312","exception":false,"start_time":"2022-08-03T14:49:52.686617","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:31:25.562035Z","iopub.execute_input":"2022-08-30T09:31:25.562598Z","iopub.status.idle":"2022-08-30T09:31:25.578499Z","shell.execute_reply.started":"2022-08-30T09:31:25.562545Z","shell.execute_reply":"2022-08-30T09:31:25.577181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_logit = sm.Logit( y_train, \n             X_train[significant_vars] ).fit()","metadata":{"papermill":{"duration":55.357353,"end_time":"2022-08-03T14:50:48.102615","exception":false,"start_time":"2022-08-03T14:49:52.745262","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:31:30.403894Z","iopub.execute_input":"2022-08-30T09:31:30.404359Z","iopub.status.idle":"2022-08-30T09:32:15.462496Z","shell.execute_reply.started":"2022-08-30T09:31:30.404322Z","shell.execute_reply":"2022-08-30T09:32:15.460742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_logit.summary2()","metadata":{"papermill":{"duration":3.800379,"end_time":"2022-08-03T14:50:51.956669","exception":false,"start_time":"2022-08-03T14:50:48.156290","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:06:13.750470Z","iopub.execute_input":"2022-08-30T09:06:13.751095Z","iopub.status.idle":"2022-08-30T09:06:17.442558Z","shell.execute_reply.started":"2022-08-30T09:06:13.751046Z","shell.execute_reply":"2022-08-30T09:06:17.441158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_df = pd.DataFrame( { \"actual\": y_test, \n                            \"predicted_prob\": final_logit.predict( \n                         X_test[significant_vars] )  } ) ","metadata":{"papermill":{"duration":0.557548,"end_time":"2022-08-03T14:50:52.534573","exception":false,"start_time":"2022-08-03T14:50:51.977025","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:32:33.728465Z","iopub.execute_input":"2022-08-30T09:32:33.728985Z","iopub.status.idle":"2022-08-30T09:32:34.331671Z","shell.execute_reply.started":"2022-08-30T09:32:33.728942Z","shell.execute_reply":"2022-08-30T09:32:34.329825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_df.sample(10, random_state = 42)","metadata":{"papermill":{"duration":0.058935,"end_time":"2022-08-03T14:50:52.650282","exception":false,"start_time":"2022-08-03T14:50:52.591347","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:32:38.188695Z","iopub.execute_input":"2022-08-30T09:32:38.189142Z","iopub.status.idle":"2022-08-30T09:32:38.227220Z","shell.execute_reply.started":"2022-08-30T09:32:38.189105Z","shell.execute_reply":"2022-08-30T09:32:38.225955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_df['predicted'] = y_pred_df.predicted_prob.map( \n                            lambda x: 1 if x > 0.5 else 0)\n\ny_pred_df.sample(10, random_state = 42)","metadata":{"papermill":{"duration":0.336637,"end_time":"2022-08-03T14:50:53.007225","exception":false,"start_time":"2022-08-03T14:50:52.670588","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:32:48.807659Z","iopub.execute_input":"2022-08-30T09:32:48.808663Z","iopub.status.idle":"2022-08-30T09:32:49.124537Z","shell.execute_reply.started":"2022-08-30T09:32:48.808621Z","shell.execute_reply":"2022-08-30T09:32:49.123305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sn\n%matplotlib inline\nfrom sklearn import metrics","metadata":{"papermill":{"duration":0.033286,"end_time":"2022-08-03T14:50:53.062097","exception":false,"start_time":"2022-08-03T14:50:53.028811","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:00.540407Z","iopub.execute_input":"2022-08-30T09:33:00.541259Z","iopub.status.idle":"2022-08-30T09:33:00.549887Z","shell.execute_reply.started":"2022-08-30T09:33:00.541217Z","shell.execute_reply":"2022-08-30T09:33:00.548730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_cm( actual, predicted ):\n    ## Cret\n    cm = metrics.confusion_matrix( actual, predicted )\n    sn.heatmap(cm, annot=True,  fmt='.2f', \n               xticklabels = [\"Bad Credit\", \"Good Credit\"] , \n               yticklabels = [\"Bad credit\", \"Good Credit\"] )\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.show()","metadata":{"papermill":{"duration":0.033383,"end_time":"2022-08-03T14:50:53.116808","exception":false,"start_time":"2022-08-03T14:50:53.083425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:24.873944Z","iopub.execute_input":"2022-08-30T09:33:24.874388Z","iopub.status.idle":"2022-08-30T09:33:24.881293Z","shell.execute_reply.started":"2022-08-30T09:33:24.874354Z","shell.execute_reply":"2022-08-30T09:33:24.879965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_cm( y_pred_df.actual, \n        y_pred_df.predicted )","metadata":{"papermill":{"duration":0.361001,"end_time":"2022-08-03T14:50:53.498696","exception":false,"start_time":"2022-08-03T14:50:53.137695","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:29.772885Z","iopub.execute_input":"2022-08-30T09:33:29.773992Z","iopub.status.idle":"2022-08-30T09:33:30.138077Z","shell.execute_reply.started":"2022-08-30T09:33:29.773935Z","shell.execute_reply":"2022-08-30T09:33:30.136246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( metrics.classification_report( y_pred_df.actual, \n                                     y_pred_df.predicted ) )","metadata":{"papermill":{"duration":1.449727,"end_time":"2022-08-03T14:50:54.970333","exception":false,"start_time":"2022-08-03T14:50:53.520606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:37.675638Z","iopub.execute_input":"2022-08-30T09:33:37.676927Z","iopub.status.idle":"2022-08-30T09:33:39.081831Z","shell.execute_reply.started":"2022-08-30T09:33:37.676862Z","shell.execute_reply":"2022-08-30T09:33:39.080774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure( figsize = (8,6) )\nsn.distplot( y_pred_df[y_pred_df.actual == 1][\"predicted_prob\"], \n            kde=False, color = 'b', \n            label = 'Bad Credit' )\nsn.distplot( y_pred_df[y_pred_df.actual == 0][\"predicted_prob\"], \n            kde=False, color = 'g', \n            label = 'Good Credit' )\nplt.legend()\nplt.show()","metadata":{"papermill":{"duration":0.434807,"end_time":"2022-08-03T14:50:55.427758","exception":false,"start_time":"2022-08-03T14:50:54.992951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:43.762800Z","iopub.execute_input":"2022-08-30T09:33:43.763232Z","iopub.status.idle":"2022-08-30T09:33:44.176483Z","shell.execute_reply.started":"2022-08-30T09:33:43.763198Z","shell.execute_reply":"2022-08-30T09:33:44.174964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_roc( actual, probs ):\n    fpr, \\\n    tpr, \\\n    thresholds = metrics.roc_curve( actual, \n                                    probs,\n                                    drop_intermediate = False )\n    auc_score = metrics.roc_auc_score( actual, probs )\n    plt.figure(figsize=(8, 6))\n    plt.plot( fpr, tpr, label='ROC curve (area = %0.2f)' % auc_score )\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate or [1 - True Negative Rate]')\n    plt.ylabel('True Positive Rate')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \n    return fpr, tpr, thresholds","metadata":{"papermill":{"duration":0.035582,"end_time":"2022-08-03T14:50:55.485630","exception":false,"start_time":"2022-08-03T14:50:55.450048","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:48.135936Z","iopub.execute_input":"2022-08-30T09:33:48.136369Z","iopub.status.idle":"2022-08-30T09:33:48.144675Z","shell.execute_reply.started":"2022-08-30T09:33:48.136331Z","shell.execute_reply":"2022-08-30T09:33:48.143633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpr, tpr, thresholds = draw_roc( y_pred_df.actual, \n                                 y_pred_df.predicted_prob)","metadata":{"papermill":{"duration":0.645412,"end_time":"2022-08-03T14:50:56.153259","exception":false,"start_time":"2022-08-03T14:50:55.507847","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:52.350619Z","iopub.execute_input":"2022-08-30T09:33:52.351054Z","iopub.status.idle":"2022-08-30T09:33:52.992729Z","shell.execute_reply.started":"2022-08-30T09:33:52.351019Z","shell.execute_reply":"2022-08-30T09:33:52.991334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auc_score = metrics.roc_auc_score( y_pred_df.actual, \n                                  y_pred_df.predicted_prob  )\nround( float( auc_score ), 2 )","metadata":{"papermill":{"duration":0.324942,"end_time":"2022-08-03T14:50:56.501851","exception":false,"start_time":"2022-08-03T14:50:56.176909","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:33:57.216596Z","iopub.execute_input":"2022-08-30T09:33:57.217301Z","iopub.status.idle":"2022-08-30T09:33:57.502205Z","shell.execute_reply.started":"2022-08-30T09:33:57.217261Z","shell.execute_reply":"2022-08-30T09:33:57.500746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tpr_fpr = pd.DataFrame( { 'tpr': tpr, \n                         'fpr': fpr, \n                         'thresholds': thresholds } )\n\ntpr_fpr['diff'] = tpr_fpr.tpr - tpr_fpr.fpr\ntpr_fpr.sort_values( 'diff', ascending = False )[0:5]","metadata":{"papermill":{"duration":0.106153,"end_time":"2022-08-03T14:50:56.630507","exception":false,"start_time":"2022-08-03T14:50:56.524354","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:34:00.761097Z","iopub.execute_input":"2022-08-30T09:34:00.761585Z","iopub.status.idle":"2022-08-30T09:34:00.834223Z","shell.execute_reply.started":"2022-08-30T09:34:00.761549Z","shell.execute_reply":"2022-08-30T09:34:00.832639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_df['predicted_new'] = y_pred_df.predicted_prob.map( \n                                lambda x: 1 if x > 0.453 else 0)","metadata":{"papermill":{"duration":0.321509,"end_time":"2022-08-03T14:50:56.974554","exception":false,"start_time":"2022-08-03T14:50:56.653045","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:34:34.709555Z","iopub.execute_input":"2022-08-30T09:34:34.710108Z","iopub.status.idle":"2022-08-30T09:34:35.019120Z","shell.execute_reply.started":"2022-08-30T09:34:34.710065Z","shell.execute_reply":"2022-08-30T09:34:35.018060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_cm( y_pred_df.actual, \n        y_pred_df.predicted_new)","metadata":{"papermill":{"duration":0.374545,"end_time":"2022-08-03T14:50:57.374228","exception":false,"start_time":"2022-08-03T14:50:56.999683","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:34:38.463979Z","iopub.execute_input":"2022-08-30T09:34:38.464426Z","iopub.status.idle":"2022-08-30T09:34:38.802482Z","shell.execute_reply.started":"2022-08-30T09:34:38.464387Z","shell.execute_reply":"2022-08-30T09:34:38.801080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.classification_report( y_pred_df.actual, \n                                     y_pred_df.predicted_new ))","metadata":{"papermill":{"duration":1.478474,"end_time":"2022-08-03T14:50:58.877431","exception":false,"start_time":"2022-08-03T14:50:57.398957","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:34:52.894647Z","iopub.execute_input":"2022-08-30T09:34:52.895212Z","iopub.status.idle":"2022-08-30T09:34:54.304996Z","shell.execute_reply.started":"2022-08-30T09:34:52.895171Z","shell.execute_reply":"2022-08-30T09:34:54.303792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del encoded_df\n\ngc.collect()","metadata":{"papermill":{"duration":0.191194,"end_time":"2022-08-03T14:50:59.093353","exception":false,"start_time":"2022-08-03T14:50:58.902159","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:35:01.389405Z","iopub.execute_input":"2022-08-30T09:35:01.389995Z","iopub.status.idle":"2022-08-30T09:35:05.220479Z","shell.execute_reply.started":"2022-08-30T09:35:01.389947Z","shell.execute_reply":"2022-08-30T09:35:05.218952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del new_df\ngc.collect()\n","metadata":{"papermill":{"duration":0.160077,"end_time":"2022-08-03T14:50:59.277989","exception":false,"start_time":"2022-08-03T14:50:59.117912","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:35:05.222382Z","iopub.execute_input":"2022-08-30T09:35:05.223536Z","iopub.status.idle":"2022-08-30T09:35:05.523237Z","shell.execute_reply.started":"2022-08-30T09:35:05.223496Z","shell.execute_reply":"2022-08-30T09:35:05.521886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train\ngc.collect()\n","metadata":{"papermill":{"duration":0.225612,"end_time":"2022-08-03T14:50:59.528144","exception":false,"start_time":"2022-08-03T14:50:59.302532","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:35:08.777173Z","iopub.execute_input":"2022-08-30T09:35:08.777649Z","iopub.status.idle":"2022-08-30T09:35:09.069890Z","shell.execute_reply.started":"2022-08-30T09:35:08.777611Z","shell.execute_reply":"2022-08-30T09:35:09.068576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del traindata_cat\ngc.collect()\n","metadata":{"papermill":{"duration":0.227287,"end_time":"2022-08-03T14:50:59.780119","exception":false,"start_time":"2022-08-03T14:50:59.552832","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:22.283852Z","iopub.execute_input":"2022-08-30T09:37:22.284388Z","iopub.status.idle":"2022-08-30T09:37:22.587603Z","shell.execute_reply.started":"2022-08-30T09:37:22.284349Z","shell.execute_reply":"2022-08-30T09:37:22.585964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del traindata_num\ngc.collect()","metadata":{"papermill":{"duration":0.229426,"end_time":"2022-08-03T14:51:00.033965","exception":false,"start_time":"2022-08-03T14:50:59.804539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:24.814857Z","iopub.execute_input":"2022-08-30T09:37:24.816223Z","iopub.status.idle":"2022-08-30T09:37:25.187030Z","shell.execute_reply.started":"2022-08-30T09:37:24.816172Z","shell.execute_reply":"2022-08-30T09:37:25.180406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del imputed_train\n\ngc.collect()","metadata":{"papermill":{"duration":0.187548,"end_time":"2022-08-03T14:51:00.246914","exception":false,"start_time":"2022-08-03T14:51:00.059366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:30.923811Z","iopub.execute_input":"2022-08-30T09:37:30.924371Z","iopub.status.idle":"2022-08-30T09:37:31.294862Z","shell.execute_reply.started":"2022-08-30T09:37:30.924326Z","shell.execute_reply":"2022-08-30T09:37:31.293351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dfnum\ndel dfcat\ngc.collect()","metadata":{"papermill":{"duration":0.246625,"end_time":"2022-08-03T14:51:00.517877","exception":false,"start_time":"2022-08-03T14:51:00.271252","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:34.248131Z","iopub.execute_input":"2022-08-30T09:37:34.248595Z","iopub.status.idle":"2022-08-30T09:37:34.535622Z","shell.execute_reply.started":"2022-08-30T09:37:34.248555Z","shell.execute_reply":"2022-08-30T09:37:34.534156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ngc.collect()","metadata":{"papermill":{"duration":0.262665,"end_time":"2022-08-03T14:51:00.806125","exception":false,"start_time":"2022-08-03T14:51:00.543460","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:36.336607Z","iopub.execute_input":"2022-08-30T09:37:36.337222Z","iopub.status.idle":"2022-08-30T09:37:36.655574Z","shell.execute_reply.started":"2022-08-30T09:37:36.337181Z","shell.execute_reply":"2022-08-30T09:37:36.654346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_columns.remove('target')\nfeature_columns","metadata":{"papermill":{"duration":0.037235,"end_time":"2022-08-03T14:51:00.869876","exception":false,"start_time":"2022-08-03T14:51:00.832641","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:39.239929Z","iopub.execute_input":"2022-08-30T09:37:39.240365Z","iopub.status.idle":"2022-08-30T09:37:39.249297Z","shell.execute_reply.started":"2022-08-30T09:37:39.240328Z","shell.execute_reply":"2022-08-30T09:37:39.248033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cid = dfn_test.customer_ID\n\n\ndfn_test = dfn_test[feature_columns]","metadata":{"papermill":{"duration":0.473436,"end_time":"2022-08-03T14:51:01.368066","exception":false,"start_time":"2022-08-03T14:51:00.894630","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:44.783183Z","iopub.execute_input":"2022-08-30T09:37:44.784420Z","iopub.status.idle":"2022-08-30T09:37:45.132693Z","shell.execute_reply.started":"2022-08-30T09:37:44.784378Z","shell.execute_reply":"2022-08-30T09:37:45.131189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns = list(set(feature_columns)-set(int_columns))","metadata":{"papermill":{"duration":0.035131,"end_time":"2022-08-03T14:51:01.429259","exception":false,"start_time":"2022-08-03T14:51:01.394128","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:47.726542Z","iopub.execute_input":"2022-08-30T09:37:47.727014Z","iopub.status.idle":"2022-08-30T09:37:47.733091Z","shell.execute_reply.started":"2022-08-30T09:37:47.726975Z","shell.execute_reply":"2022-08-30T09:37:47.731953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns","metadata":{"papermill":{"duration":0.035406,"end_time":"2022-08-03T14:51:01.489943","exception":false,"start_time":"2022-08-03T14:51:01.454537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:50.521829Z","iopub.execute_input":"2022-08-30T09:37:50.522342Z","iopub.status.idle":"2022-08-30T09:37:50.531763Z","shell.execute_reply.started":"2022-08-30T09:37:50.522306Z","shell.execute_reply":"2022-08-30T09:37:50.530051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create two DataFrames, one for each data type\ndata_numeric = dfn_test[int_columns]\ndata_categorical = pd.DataFrame(dfn_test[cat_columns])","metadata":{"papermill":{"duration":0.353474,"end_time":"2022-08-03T14:51:01.868345","exception":false,"start_time":"2022-08-03T14:51:01.514871","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:37:57.990665Z","iopub.execute_input":"2022-08-30T09:37:57.991198Z","iopub.status.idle":"2022-08-30T09:37:58.325465Z","shell.execute_reply.started":"2022-08-30T09:37:57.991156Z","shell.execute_reply":"2022-08-30T09:37:58.323993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = dfn_test.dropna()\n\nfor col in int_columns:\n    data_numeric[col] = data_numeric[col].fillna(test[col].median())\ndel test\ngc.collect()","metadata":{"papermill":{"duration":3.232999,"end_time":"2022-08-03T14:51:05.127141","exception":false,"start_time":"2022-08-03T14:51:01.894142","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:01.501408Z","iopub.execute_input":"2022-08-30T09:38:01.501829Z","iopub.status.idle":"2022-08-30T09:38:04.916119Z","shell.execute_reply.started":"2022-08-30T09:38:01.501795Z","shell.execute_reply":"2022-08-30T09:38:04.914946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_columns:\n    data_categorical[col] = data_categorical[col].fillna(data_categorical[col].value_counts().idxmax())\n\n","metadata":{"papermill":{"duration":0.112036,"end_time":"2022-08-03T14:51:05.265998","exception":false,"start_time":"2022-08-03T14:51:05.153962","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:06.539497Z","iopub.execute_input":"2022-08-30T09:38:06.539954Z","iopub.status.idle":"2022-08-30T09:38:06.628308Z","shell.execute_reply.started":"2022-08-30T09:38:06.539886Z","shell.execute_reply":"2022-08-30T09:38:06.627050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.concat([data_categorical, data_numeric], axis=1, join='inner')\nresult","metadata":{"papermill":{"duration":0.438495,"end_time":"2022-08-03T14:51:05.731315","exception":false,"start_time":"2022-08-03T14:51:05.292820","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:09.200059Z","iopub.execute_input":"2022-08-30T09:38:09.200631Z","iopub.status.idle":"2022-08-30T09:38:09.625083Z","shell.execute_reply.started":"2022-08-30T09:38:09.200578Z","shell.execute_reply":"2022-08-30T09:38:09.623863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_dft = pd.get_dummies( result, \n                                    columns = cat_columns,\n                                    drop_first = True )\n\nencoded_dft","metadata":{"papermill":{"duration":1.094652,"end_time":"2022-08-03T14:51:06.853843","exception":false,"start_time":"2022-08-03T14:51:05.759191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:16.731586Z","iopub.execute_input":"2022-08-30T09:38:16.732619Z","iopub.status.idle":"2022-08-30T09:38:17.795364Z","shell.execute_reply.started":"2022-08-30T09:38:16.732579Z","shell.execute_reply":"2022-08-30T09:38:17.794143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(encoded_dft.columns) - set(significant_vars)","metadata":{"papermill":{"duration":0.038338,"end_time":"2022-08-03T14:51:06.919956","exception":false,"start_time":"2022-08-03T14:51:06.881618","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:22.177725Z","iopub.execute_input":"2022-08-30T09:38:22.178217Z","iopub.status.idle":"2022-08-30T09:38:22.187510Z","shell.execute_reply.started":"2022-08-30T09:38:22.178177Z","shell.execute_reply":"2022-08-30T09:38:22.186478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(significant_vars)","metadata":{"papermill":{"duration":0.037804,"end_time":"2022-08-03T14:51:06.985939","exception":false,"start_time":"2022-08-03T14:51:06.948135","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:25.586670Z","iopub.execute_input":"2022-08-30T09:38:25.587193Z","iopub.status.idle":"2022-08-30T09:38:25.594334Z","shell.execute_reply.started":"2022-08-30T09:38:25.587153Z","shell.execute_reply":"2022-08-30T09:38:25.593284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = encoded_dft[significant_vars]","metadata":{"papermill":{"duration":0.452614,"end_time":"2022-08-03T14:51:07.466455","exception":false,"start_time":"2022-08-03T14:51:07.013841","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:29.694367Z","iopub.execute_input":"2022-08-30T09:38:29.694776Z","iopub.status.idle":"2022-08-30T09:38:30.154259Z","shell.execute_reply.started":"2022-08-30T09:38:29.694743Z","shell.execute_reply":"2022-08-30T09:38:30.152868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(X_test.columns) - set(significant_vars)","metadata":{"papermill":{"duration":0.039711,"end_time":"2022-08-03T14:51:07.534037","exception":false,"start_time":"2022-08-03T14:51:07.494326","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:35.237507Z","iopub.execute_input":"2022-08-30T09:38:35.237978Z","iopub.status.idle":"2022-08-30T09:38:35.245504Z","shell.execute_reply.started":"2022-08-30T09:38:35.237927Z","shell.execute_reply":"2022-08-30T09:38:35.244504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = final_logit.predict( X_test  )","metadata":{"papermill":{"duration":0.46379,"end_time":"2022-08-03T14:51:08.024816","exception":false,"start_time":"2022-08-03T14:51:07.561026","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:40.485581Z","iopub.execute_input":"2022-08-30T09:38:40.486038Z","iopub.status.idle":"2022-08-30T09:38:41.153069Z","shell.execute_reply.started":"2022-08-30T09:38:40.486000Z","shell.execute_reply":"2022-08-30T09:38:41.151313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"papermill":{"duration":0.0403,"end_time":"2022-08-03T14:51:08.134173","exception":false,"start_time":"2022-08-03T14:51:08.093873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:38:46.932152Z","iopub.execute_input":"2022-08-30T09:38:46.932646Z","iopub.status.idle":"2022-08-30T09:38:46.944409Z","shell.execute_reply.started":"2022-08-30T09:38:46.932608Z","shell.execute_reply":"2022-08-30T09:38:46.942530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(cid)\nsubmission['prediction'] = preds","metadata":{"papermill":{"duration":0.057545,"end_time":"2022-08-03T14:51:08.219903","exception":false,"start_time":"2022-08-03T14:51:08.162358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:39:00.116978Z","iopub.execute_input":"2022-08-30T09:39:00.117450Z","iopub.status.idle":"2022-08-30T09:39:00.147251Z","shell.execute_reply.started":"2022-08-30T09:39:00.117413Z","shell.execute_reply":"2022-08-30T09:39:00.146308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"papermill":{"duration":0.044138,"end_time":"2022-08-03T14:51:08.291465","exception":false,"start_time":"2022-08-03T14:51:08.247327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:39:02.976878Z","iopub.execute_input":"2022-08-30T09:39:02.978287Z","iopub.status.idle":"2022-08-30T09:39:03.002732Z","shell.execute_reply.started":"2022-08-30T09:39:02.978229Z","shell.execute_reply":"2022-08-30T09:39:03.001170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"papermill":{"duration":3.273266,"end_time":"2022-08-03T14:51:11.591882","exception":false,"start_time":"2022-08-03T14:51:08.318616","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T09:39:27.991120Z","iopub.execute_input":"2022-08-30T09:39:27.991653Z","iopub.status.idle":"2022-08-30T09:39:31.395342Z","shell.execute_reply.started":"2022-08-30T09:39:27.991602Z","shell.execute_reply":"2022-08-30T09:39:31.394318Z"},"trusted":true},"execution_count":null,"outputs":[]}]}