{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# __American Express - Default Prediction__","metadata":{}},{"cell_type":"markdown","source":"### Include the required library","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport sklearn\nfrom dask import dataframe as dd","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:22:37.732012Z","iopub.execute_input":"2022-11-18T17:22:37.732985Z","iopub.status.idle":"2022-11-18T17:22:40.821551Z","shell.execute_reply.started":"2022-11-18T17:22:37.732938Z","shell.execute_reply":"2022-11-18T17:22:40.820550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### __Quick Overview:__\n#### I needed to save time and ram space in order to calculate the model score due to the large volume of data.One method I used to use was to randomly select a portion of the data  that was as large as my ram space or any platform could handle. Then, I used multiple techniques to identify the features that were most important to my model and read the entire data with those features.\n#### Additionally, Models have increasing risk of overfitting with increasing number of features.","metadata":{}},{"cell_type":"markdown","source":"### I read the 500000 rows of data to find the most relevant and effective features.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/amex-default-prediction/train_data.csv\", nrows=500000)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T21:57:58.942140Z","iopub.execute_input":"2022-11-17T21:57:58.942907Z","iopub.status.idle":"2022-11-17T21:58:37.164478Z","shell.execute_reply.started":"2022-11-17T21:57:58.942855Z","shell.execute_reply":"2022-11-17T21:58:37.163451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### I read the Train_label target.","metadata":{}},{"cell_type":"code","source":"df_target = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:34:05.145003Z","iopub.execute_input":"2022-11-17T22:34:05.145388Z","iopub.status.idle":"2022-11-17T22:34:06.211732Z","shell.execute_reply.started":"2022-11-17T22:34:05.145353Z","shell.execute_reply":"2022-11-17T22:34:06.210652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### shuffling the data","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import shuffle","metadata":{"execution":{"iopub.status.busy":"2022-11-17T23:02:39.493175Z","iopub.execute_input":"2022-11-17T23:02:39.493874Z","iopub.status.idle":"2022-11-17T23:02:39.499432Z","shell.execute_reply.started":"2022-11-17T23:02:39.493832Z","shell.execute_reply":"2022-11-17T23:02:39.498257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_train_shuffle = shuffle(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T21:59:51.074712Z","iopub.execute_input":"2022-11-17T21:59:51.075071Z","iopub.status.idle":"2022-11-17T21:59:51.793163Z","shell.execute_reply.started":"2022-11-17T21:59:51.075021Z","shell.execute_reply":"2022-11-17T21:59:51.792102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Merging two sets of data","metadata":{}},{"cell_type":"code","source":"result =  pd.merge(df_train_shuffle, df_target, on='customer_ID', how='inner')","metadata":{"execution":{"iopub.status.busy":"2022-11-17T21:59:54.181931Z","iopub.execute_input":"2022-11-17T21:59:54.182310Z","iopub.status.idle":"2022-11-17T21:59:56.610131Z","shell.execute_reply.started":"2022-11-17T21:59:54.182278Z","shell.execute_reply":"2022-11-17T21:59:56.609167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T21:59:57.845808Z","iopub.execute_input":"2022-11-17T21:59:57.846213Z","iopub.status.idle":"2022-11-17T21:59:57.877431Z","shell.execute_reply.started":"2022-11-17T21:59:57.846178Z","shell.execute_reply":"2022-11-17T21:59:57.876346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### shuffles once more","metadata":{}},{"cell_type":"code","source":"result_shuffle = shuffle(result)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:00.495894Z","iopub.execute_input":"2022-11-17T22:00:00.496617Z","iopub.status.idle":"2022-11-17T22:00:02.012106Z","shell.execute_reply.started":"2022-11-17T22:00:00.496580Z","shell.execute_reply":"2022-11-17T22:00:02.009894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_shuffle.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:02.014404Z","iopub.execute_input":"2022-11-17T22:00:02.015333Z","iopub.status.idle":"2022-11-17T22:00:02.041889Z","shell.execute_reply.started":"2022-11-17T22:00:02.015286Z","shell.execute_reply":"2022-11-17T22:00:02.040981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### copy the data that was shuffled into temporary data.","metadata":{}},{"cell_type":"code","source":"df_tmp = result_shuffle.copy()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:04.229574Z","iopub.execute_input":"2022-11-17T22:00:04.229926Z","iopub.status.idle":"2022-11-17T22:00:04.511107Z","shell.execute_reply.started":"2022-11-17T22:00:04.229895Z","shell.execute_reply":"2022-11-17T22:00:04.510098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Decoding features of time","metadata":{}},{"cell_type":"code","source":"df_tmp['S_2'] = pd.to_datetime(df_tmp['S_2'], errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:06.204568Z","iopub.execute_input":"2022-11-17T22:00:06.205297Z","iopub.status.idle":"2022-11-17T22:00:06.326007Z","shell.execute_reply.started":"2022-11-17T22:00:06.205259Z","shell.execute_reply":"2022-11-17T22:00:06.325078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp[\"Year\"]= df_tmp.S_2.dt.year\ndf_tmp[\"Month\"]= df_tmp.S_2.dt.month\ndf_tmp[\"Day\"]= df_tmp.S_2.dt.day","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:16.707445Z","iopub.execute_input":"2022-11-17T22:00:16.707809Z","iopub.status.idle":"2022-11-17T22:00:16.857764Z","shell.execute_reply.started":"2022-11-17T22:00:16.707777Z","shell.execute_reply":"2022-11-17T22:00:16.856812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:18.356675Z","iopub.execute_input":"2022-11-17T22:00:18.357030Z","iopub.status.idle":"2022-11-17T22:00:18.384984Z","shell.execute_reply.started":"2022-11-17T22:00:18.357001Z","shell.execute_reply":"2022-11-17T22:00:18.384061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp_1 = df_tmp.drop([\"customer_ID\",\"S_2\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:21.882464Z","iopub.execute_input":"2022-11-17T22:00:21.882828Z","iopub.status.idle":"2022-11-17T22:00:22.118323Z","shell.execute_reply.started":"2022-11-17T22:00:21.882797Z","shell.execute_reply":"2022-11-17T22:00:22.117337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp_1.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:35.038254Z","iopub.execute_input":"2022-11-17T22:00:35.038603Z","iopub.status.idle":"2022-11-17T22:00:35.064566Z","shell.execute_reply.started":"2022-11-17T22:00:35.038573Z","shell.execute_reply":"2022-11-17T22:00:35.063097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## __EDA__\n","metadata":{}},{"cell_type":"markdown","source":"### Converting string to categories\n","metadata":{}},{"cell_type":"code","source":"for label, content in df_tmp_1.items():\n    if pd.api.types.is_string_dtype(content):\n        df_tmp_1[label]= content.astype(\"category\").cat.as_unordered()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:00:39.522535Z","iopub.execute_input":"2022-11-17T22:00:39.522890Z","iopub.status.idle":"2022-11-17T22:00:39.594268Z","shell.execute_reply.started":"2022-11-17T22:00:39.522857Z","shell.execute_reply":"2022-11-17T22:00:39.593358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filling missing value","metadata":{}},{"cell_type":"code","source":"for label,content in df_tmp_1.items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():  \n            df_tmp_1[label]= content.fillna(content.median())","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:22.645176Z","iopub.execute_input":"2022-11-17T22:01:22.645554Z","iopub.status.idle":"2022-11-17T22:01:23.988400Z","shell.execute_reply.started":"2022-11-17T22:01:22.645522Z","shell.execute_reply":"2022-11-17T22:01:23.987240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_tmp_1.items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:27.356941Z","iopub.execute_input":"2022-11-17T22:01:27.357302Z","iopub.status.idle":"2022-11-17T22:01:27.569904Z","shell.execute_reply.started":"2022-11-17T22:01:27.357269Z","shell.execute_reply":"2022-11-17T22:01:27.568920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Filling and turning categorical variable into numbers\nfor label,content in df_tmp_1.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        print(label)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:29.521443Z","iopub.execute_input":"2022-11-17T22:01:29.521807Z","iopub.status.idle":"2022-11-17T22:01:29.529208Z","shell.execute_reply.started":"2022-11-17T22:01:29.521775Z","shell.execute_reply":"2022-11-17T22:01:29.528001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_tmp_1.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        #Add binary column to indicate whether sample had missing value\n        \n        # Turn categories into numbers and add +1\n        df_tmp_1[label] =pd.Categorical(content).codes + 1","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:33.011870Z","iopub.execute_input":"2022-11-17T22:01:33.012269Z","iopub.status.idle":"2022-11-17T22:01:33.027582Z","shell.execute_reply.started":"2022-11-17T22:01:33.012233Z","shell.execute_reply":"2022-11-17T22:01:33.026607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_tmp_1.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:33.029839Z","iopub.execute_input":"2022-11-17T22:01:33.030687Z","iopub.status.idle":"2022-11-17T22:01:33.038094Z","shell.execute_reply.started":"2022-11-17T22:01:33.030658Z","shell.execute_reply":"2022-11-17T22:01:33.037034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp_1.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:33.550774Z","iopub.execute_input":"2022-11-17T22:01:33.551149Z","iopub.status.idle":"2022-11-17T22:01:33.579088Z","shell.execute_reply.started":"2022-11-17T22:01:33.551117Z","shell.execute_reply":"2022-11-17T22:01:33.578205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_tmp_1_without_target = df_tmp_1.drop([\"target\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:34.888131Z","iopub.execute_input":"2022-11-17T22:01:34.888792Z","iopub.status.idle":"2022-11-17T22:01:35.142348Z","shell.execute_reply.started":"2022-11-17T22:01:34.888756Z","shell.execute_reply":"2022-11-17T22:01:35.141278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### __The mutual_info_classif was used to find useful features.__","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_classif\nthreshold = 30  # the number of most relevant features\nhigh_score_features = []\nfeature_scores = mutual_info_classif(f_tmp_1_without_target, df_tmp_1.target, random_state=0)\nfor score, f_name in sorted(zip(feature_scores, f_tmp_1_without_target.columns), reverse=True)[:threshold]:\n        print(f_name, score)\n        high_score_features.append(f_name)\ndf_norm_mic = f_tmp_1_without_target[high_score_features]\nprint(df_norm_mic.columns)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:01:37.468829Z","iopub.execute_input":"2022-11-17T22:01:37.469212Z","iopub.status.idle":"2022-11-17T22:10:59.694787Z","shell.execute_reply.started":"2022-11-17T22:01:37.469177Z","shell.execute_reply":"2022-11-17T22:10:59.693616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# __We will read the training data set that contains selected features in accordance with previous analysis.__","metadata":{}},{"cell_type":"markdown","source":"# train_data containing useful features","metadata":{}},{"cell_type":"code","source":"effective_col = ['customer_ID','P_2', 'D_48', 'D_61', 'B_9', 'B_18', 'D_44', 'D_75', 'B_7', 'B_23',\n       'B_6', 'D_62', 'B_10', 'B_3', 'B_2', 'B_1', 'B_37', 'B_11', 'D_55',\n       'D_74', 'B_38', 'D_58', 'B_40', 'B_4', 'B_20', 'B_33']","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:34:16.172744Z","iopub.execute_input":"2022-11-17T22:34:16.173147Z","iopub.status.idle":"2022-11-17T22:34:16.178883Z","shell.execute_reply.started":"2022-11-17T22:34:16.173111Z","shell.execute_reply":"2022-11-17T22:34:16.177610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_target = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:34:17.954418Z","iopub.execute_input":"2022-11-17T22:34:17.954788Z","iopub.status.idle":"2022-11-17T22:34:18.410862Z","shell.execute_reply.started":"2022-11-17T22:34:17.954756Z","shell.execute_reply":"2022-11-17T22:34:18.409852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new = pd.read_csv('../input/amex-default-prediction/train_data.csv', usecols = effective_col)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:34:20.986191Z","iopub.execute_input":"2022-11-17T22:34:20.986597Z","iopub.status.idle":"2022-11-17T22:39:01.484361Z","shell.execute_reply.started":"2022-11-17T22:34:20.986561Z","shell.execute_reply":"2022-11-17T22:39:01.483228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:16.038889Z","iopub.execute_input":"2022-11-17T22:40:16.039685Z","iopub.status.idle":"2022-11-17T22:40:16.085081Z","shell.execute_reply.started":"2022-11-17T22:40:16.039643Z","shell.execute_reply":"2022-11-17T22:40:16.084106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import shuffle","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:17.165598Z","iopub.execute_input":"2022-11-17T22:40:17.167903Z","iopub.status.idle":"2022-11-17T22:40:17.177081Z","shell.execute_reply.started":"2022-11-17T22:40:17.167848Z","shell.execute_reply":"2022-11-17T22:40:17.175826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### shuffling new data","metadata":{}},{"cell_type":"code","source":"df_new_shuffle = shuffle(df_new)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:18.879866Z","iopub.execute_input":"2022-11-17T22:40:18.880255Z","iopub.status.idle":"2022-11-17T22:40:21.547262Z","shell.execute_reply.started":"2022-11-17T22:40:18.880220Z","shell.execute_reply":"2022-11-17T22:40:21.546208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new_shuffle.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:21.550479Z","iopub.execute_input":"2022-11-17T22:40:21.551232Z","iopub.status.idle":"2022-11-17T22:40:21.576683Z","shell.execute_reply.started":"2022-11-17T22:40:21.551190Z","shell.execute_reply":"2022-11-17T22:40:21.575627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining two sets of data","metadata":{}},{"cell_type":"code","source":"result_new =  pd.merge(df_new_shuffle, df_target, on='customer_ID', how='inner')","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:24.999937Z","iopub.execute_input":"2022-11-17T22:40:25.000742Z","iopub.status.idle":"2022-11-17T22:40:29.398410Z","shell.execute_reply.started":"2022-11-17T22:40:25.000702Z","shell.execute_reply":"2022-11-17T22:40:29.397346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:29.400426Z","iopub.execute_input":"2022-11-17T22:40:29.400867Z","iopub.status.idle":"2022-11-17T22:40:29.428512Z","shell.execute_reply.started":"2022-11-17T22:40:29.400800Z","shell.execute_reply":"2022-11-17T22:40:29.427548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle = shuffle(result_new)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:30.610563Z","iopub.execute_input":"2022-11-17T22:40:30.610923Z","iopub.status.idle":"2022-11-17T22:40:33.868472Z","shell.execute_reply.started":"2022-11-17T22:40:30.610890Z","shell.execute_reply":"2022-11-17T22:40:33.867371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:33.870164Z","iopub.execute_input":"2022-11-17T22:40:33.870568Z","iopub.status.idle":"2022-11-17T22:40:33.898478Z","shell.execute_reply.started":"2022-11-17T22:40:33.870536Z","shell.execute_reply":"2022-11-17T22:40:33.897404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"for label, content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if pd.api.types.is_string_dtype(content):\n        result_new_shuffle[label]= content.astype(\"category\").cat.as_unordered()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:35.981777Z","iopub.execute_input":"2022-11-17T22:40:35.982504Z","iopub.status.idle":"2022-11-17T22:40:37.119685Z","shell.execute_reply.started":"2022-11-17T22:40:35.982465Z","shell.execute_reply":"2022-11-17T22:40:37.118752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:38.081769Z","iopub.execute_input":"2022-11-17T22:40:38.082567Z","iopub.status.idle":"2022-11-17T22:40:38.111622Z","shell.execute_reply.started":"2022-11-17T22:40:38.082515Z","shell.execute_reply":"2022-11-17T22:40:38.109274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():  \n            result_new_shuffle[label]= content.fillna(content.median())","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:38.921097Z","iopub.execute_input":"2022-11-17T22:40:38.921797Z","iopub.status.idle":"2022-11-17T22:40:42.200845Z","shell.execute_reply.started":"2022-11-17T22:40:38.921760Z","shell.execute_reply":"2022-11-17T22:40:42.199812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:42.980333Z","iopub.execute_input":"2022-11-17T22:40:42.980700Z","iopub.status.idle":"2022-11-17T22:40:43.588309Z","shell.execute_reply.started":"2022-11-17T22:40:42.980669Z","shell.execute_reply":"2022-11-17T22:40:43.587334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Filling and turning categorical variable into numbers\nfor label,content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if not pd.api.types.is_numeric_dtype(content):\n        print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:44.379494Z","iopub.execute_input":"2022-11-17T22:40:44.379851Z","iopub.status.idle":"2022-11-17T22:40:44.753919Z","shell.execute_reply.started":"2022-11-17T22:40:44.379819Z","shell.execute_reply":"2022-11-17T22:40:44.752911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if not pd.api.types.is_numeric_dtype(content):\n        # Turn categories into numbers and add +1\n        result_new_shuffle[label] =pd.Categorical(content).codes + 1","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:45.219448Z","iopub.execute_input":"2022-11-17T22:40:45.220546Z","iopub.status.idle":"2022-11-17T22:40:45.585471Z","shell.execute_reply.started":"2022-11-17T22:40:45.220498Z","shell.execute_reply":"2022-11-17T22:40:45.584236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:46.064483Z","iopub.execute_input":"2022-11-17T22:40:46.065323Z","iopub.status.idle":"2022-11-17T22:40:46.093412Z","shell.execute_reply.started":"2022-11-17T22:40:46.065245Z","shell.execute_reply":"2022-11-17T22:40:46.092304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in result_new_shuffle.loc[ : , result_new_shuffle.columns != 'customer_ID'].items():\n    if not pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:46.769448Z","iopub.execute_input":"2022-11-17T22:40:46.771517Z","iopub.status.idle":"2022-11-17T22:40:47.145201Z","shell.execute_reply.started":"2022-11-17T22:40:46.771485Z","shell.execute_reply":"2022-11-17T22:40:47.144006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle_new_with_out_target = result_new_shuffle.drop([\"target\",\"customer_ID\"],axis =1)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:47.560759Z","iopub.execute_input":"2022-11-17T22:40:47.561419Z","iopub.status.idle":"2022-11-17T22:40:47.901894Z","shell.execute_reply.started":"2022-11-17T22:40:47.561371Z","shell.execute_reply":"2022-11-17T22:40:47.900429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_new_shuffle_new_with_out_target.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:48.342360Z","iopub.execute_input":"2022-11-17T22:40:48.342711Z","iopub.status.idle":"2022-11-17T22:40:48.369354Z","shell.execute_reply.started":"2022-11-17T22:40:48.342679Z","shell.execute_reply":"2022-11-17T22:40:48.368224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# split data into train/validation sets","metadata":{}},{"cell_type":"code","source":"#from sklearn.calibration import CalibratedClassifierCV\n\nfrom sklearn import linear_model\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:50.079527Z","iopub.execute_input":"2022-11-17T22:40:50.079894Z","iopub.status.idle":"2022-11-17T22:40:50.177826Z","shell.execute_reply.started":"2022-11-17T22:40:50.079863Z","shell.execute_reply":"2022-11-17T22:40:50.176808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clf = linear_model.SGDClassifier(shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:51.835287Z","iopub.execute_input":"2022-11-17T22:40:51.835921Z","iopub.status.idle":"2022-11-17T22:40:51.840987Z","shell.execute_reply.started":"2022-11-17T22:40:51.835882Z","shell.execute_reply":"2022-11-17T22:40:51.839719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#model = CalibratedClassifierCV(clf)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:54.469542Z","iopub.execute_input":"2022-11-17T22:40:54.469910Z","iopub.status.idle":"2022-11-17T22:40:54.474533Z","shell.execute_reply.started":"2022-11-17T22:40:54.469876Z","shell.execute_reply":"2022-11-17T22:40:54.473517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fraction = 0.7\nsplit_point = int(train_fraction *len(result_new_shuffle)) # (len(X) and len(y) are the same anyway)\nX_train = result_new_shuffle_new_with_out_target[0:split_point]\nX_valid = result_new_shuffle_new_with_out_target[split_point:]\n\ny_train= result_new_shuffle[\"target\"][0:split_point]\ny_valid= result_new_shuffle[\"target\"][split_point:]","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:55.670508Z","iopub.execute_input":"2022-11-17T22:40:55.670868Z","iopub.status.idle":"2022-11-17T22:40:55.677367Z","shell.execute_reply.started":"2022-11-17T22:40:55.670837Z","shell.execute_reply":"2022-11-17T22:40:55.676112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport lightgbm as lgb\nmodel = lgb.LGBMClassifier(learning_rate=0.04,max_depth=-5,random_state=42,num_leaves = 70)\nmodel.fit(X_train,y_train,eval_set=[(X_valid,y_valid),(X_train,y_train)],\n          verbose=1,eval_metric='merror')","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:40:57.114626Z","iopub.execute_input":"2022-11-17T22:40:57.115025Z","iopub.status.idle":"2022-11-17T22:43:26.111851Z","shell.execute_reply.started":"2022-11-17T22:40:57.114983Z","shell.execute_reply":"2022-11-17T22:43:26.111029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import classification_report\ny_pred_val_lgbm = model.predict(X_valid)\nprint('Accuracy on Validation set :',accuracy_score(y_valid, y_pred_val_lgbm))\nprint(\"\\n\")\nprint(classification_report(y_valid, y_pred_val_lgbm))","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:43:30.273150Z","iopub.execute_input":"2022-11-17T22:43:30.274235Z","iopub.status.idle":"2022-11-17T22:43:40.477704Z","shell.execute_reply.started":"2022-11-17T22:43:30.274180Z","shell.execute_reply":"2022-11-17T22:43:40.476450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training accuracy {:.4f}'.format(model.score(X_train,y_train)))","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:43:43.787016Z","iopub.execute_input":"2022-11-17T22:43:43.787794Z","iopub.status.idle":"2022-11-17T22:44:02.947667Z","shell.execute_reply.started":"2022-11-17T22:43:43.787754Z","shell.execute_reply":"2022-11-17T22:44:02.946364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training accuracy {:.4f}'.format(model.score(X_valid,y_valid)))","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:03.996030Z","iopub.execute_input":"2022-11-17T22:44:03.996429Z","iopub.status.idle":"2022-11-17T22:44:11.630679Z","shell.execute_reply.started":"2022-11-17T22:44:03.996396Z","shell.execute_reply":"2022-11-17T22:44:11.629643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(10, 20))\nlgb.plot_importance(model, ax=ax)\nlgb.plot_metric(model)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:12.961650Z","iopub.execute_input":"2022-11-17T22:44:12.962019Z","iopub.status.idle":"2022-11-17T22:44:13.618285Z","shell.execute_reply.started":"2022-11-17T22:44:12.961987Z","shell.execute_reply":"2022-11-17T22:44:13.617022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb.plot_tree(model,figsize=(30,40))","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:15.297444Z","iopub.execute_input":"2022-11-17T22:44:15.297813Z","iopub.status.idle":"2022-11-17T22:44:19.678320Z","shell.execute_reply.started":"2022-11-17T22:44:15.297783Z","shell.execute_reply":"2022-11-17T22:44:19.674411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn.metrics as metrics\nmetrics.plot_confusion_matrix(model,X_valid,y_valid,cmap='Blues_r')","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:22.297734Z","iopub.execute_input":"2022-11-17T22:44:22.298148Z","iopub.status.idle":"2022-11-17T22:44:30.372428Z","shell.execute_reply.started":"2022-11-17T22:44:22.298109Z","shell.execute_reply":"2022-11-17T22:44:30.371293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error, mean_absolute_error, r2_score\n\ndef rmsle(y_test,y_preds):\n    return np.sqrt(mean_squared_log_error(y_test,y_preds))","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:31.644789Z","iopub.execute_input":"2022-11-17T22:44:31.645183Z","iopub.status.idle":"2022-11-17T22:44:31.651284Z","shell.execute_reply.started":"2022-11-17T22:44:31.645148Z","shell.execute_reply":"2022-11-17T22:44:31.649797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_score(model):\n    train_preds = model.predict(X_train)\n    val_preds = model.predict(X_valid)\n    scores = {\"Train MAE\":mean_absolute_error(y_train,train_preds),\n             \"valid MAE\": mean_absolute_error(y_valid, val_preds),\n             \"trianing RMSLE\": rmsle(y_train, train_preds),\n             \"valid RMSLE\": rmsle(y_valid, val_preds),\n             \"training R^2\": r2_score(y_train,train_preds),\n             \"valid R^2\" : r2_score(y_valid,val_preds)}\n    return scores","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:33.791574Z","iopub.execute_input":"2022-11-17T22:44:33.792148Z","iopub.status.idle":"2022-11-17T22:44:33.798371Z","shell.execute_reply.started":"2022-11-17T22:44:33.792108Z","shell.execute_reply":"2022-11-17T22:44:33.797126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_score(model)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:44:35.210347Z","iopub.execute_input":"2022-11-17T22:44:35.211333Z","iopub.status.idle":"2022-11-17T22:45:01.089152Z","shell.execute_reply.started":"2022-11-17T22:44:35.211298Z","shell.execute_reply":"2022-11-17T22:45:01.088057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle","metadata":{"execution":{"iopub.status.busy":"2022-11-17T23:07:00.910658Z","iopub.execute_input":"2022-11-17T23:07:00.911459Z","iopub.status.idle":"2022-11-17T23:07:00.916168Z","shell.execute_reply.started":"2022-11-17T23:07:00.911417Z","shell.execute_reply":"2022-11-17T23:07:00.915125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# save the model to disk\nfilename = 'finalized_model.sav'\npickle.dump(model, open(filename, 'wb'))\n \n","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:45:03.423305Z","iopub.execute_input":"2022-11-17T22:45:03.423735Z","iopub.status.idle":"2022-11-17T22:45:03.469881Z","shell.execute_reply.started":"2022-11-17T22:45:03.423698Z","shell.execute_reply":"2022-11-17T22:45:03.469067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict_proba(X_valid)\nV_Frame = pd.DataFrame(predictions)\nV_Frame","metadata":{"execution":{"iopub.status.busy":"2022-11-17T22:45:04.722541Z","iopub.execute_input":"2022-11-17T22:45:04.722964Z","iopub.status.idle":"2022-11-17T22:45:11.839991Z","shell.execute_reply.started":"2022-11-17T22:45:04.722931Z","shell.execute_reply":"2022-11-17T22:45:11.838868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing Data","metadata":{}},{"cell_type":"code","source":"effective_col_test = ['P_2', 'D_48', 'D_61', 'B_9', 'B_18', 'D_44', 'D_75', 'B_7', 'B_23',\n       'B_6', 'D_62', 'B_10', 'B_3', 'B_2', 'B_1', 'B_37', 'B_11', 'D_55',\n       'D_74', 'B_38', 'D_58', 'B_40', 'B_4', 'B_20', 'B_33']","metadata":{"execution":{"iopub.status.busy":"2022-11-18T17:22:50.670510Z","iopub.execute_input":"2022-11-18T17:22:50.670867Z","iopub.status.idle":"2022-11-18T17:22:50.676712Z","shell.execute_reply.started":"2022-11-18T17:22:50.670837Z","shell.execute_reply":"2022-11-18T17:22:50.675791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_1 = pd.read_csv(\"../input/amex-default-prediction/test_data.csv\",usecols = [\"customer_ID\"],chunksize=1000000)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:23:20.351814Z","iopub.execute_input":"2022-11-18T18:23:20.352361Z","iopub.status.idle":"2022-11-18T18:23:20.361522Z","shell.execute_reply.started":"2022-11-18T18:23:20.352323Z","shell.execute_reply":"2022-11-18T18:23:20.360214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/amex-default-prediction/test_data.csv\", usecols = effective_col_test,chunksize=1000000)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:23:17.468492Z","iopub.execute_input":"2022-11-18T18:23:17.468899Z","iopub.status.idle":"2022-11-18T18:23:17.487424Z","shell.execute_reply.started":"2022-11-18T18:23:17.468866Z","shell.execute_reply":"2022-11-18T18:23:17.486517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_list = []  # append each chunk df here \n\n# Each chunk is in df format\nfor chunk in df_test:  \n    # Once the data filtering is done, append the chunk to list\n    chunk_list.append(chunk)\n    \n# concat the list into dataframe \ndf_concat = pd.concat(chunk_list)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:12:25.728312Z","iopub.execute_input":"2022-11-18T18:12:25.728681Z","iopub.status.idle":"2022-11-18T18:20:55.640078Z","shell.execute_reply.started":"2022-11-18T18:12:25.728648Z","shell.execute_reply":"2022-11-18T18:20:55.639052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_list_1 = []  # append each chunk df here \n\n# Each chunk is in df format\nfor chunk_1 in df_test_1:  \n\n    # Once the data filtering is done, append the chunk to list\n    chunk_list_1.append(chunk_1)\n    \n# concat the list into dataframe \ndf_concat_1 = pd.concat(chunk_list_1)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:23:24.913570Z","iopub.execute_input":"2022-11-18T18:23:24.913916Z","iopub.status.idle":"2022-11-18T18:34:16.872573Z","shell.execute_reply.started":"2022-11-18T18:23:24.913888Z","shell.execute_reply":"2022-11-18T18:34:16.871534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_concat.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T19:00:05.667669Z","iopub.execute_input":"2022-11-18T19:00:05.669017Z","iopub.status.idle":"2022-11-18T19:00:05.753056Z","shell.execute_reply.started":"2022-11-18T19:00:05.668976Z","shell.execute_reply":"2022-11-18T19:00:05.752027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label, content in df_concat.items():\n    if pd.api.types.is_string_dtype(content):\n        df_concat[label]= content.astype(\"category\").cat.as_unordered()","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:02.964195Z","iopub.execute_input":"2022-11-18T18:37:02.964887Z","iopub.status.idle":"2022-11-18T18:37:02.978644Z","shell.execute_reply.started":"2022-11-18T18:37:02.964851Z","shell.execute_reply":"2022-11-18T18:37:02.977725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_concat.items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():  \n            df_concat[label]= content.fillna(content.median())","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:13.490186Z","iopub.execute_input":"2022-11-18T18:37:13.490592Z","iopub.status.idle":"2022-11-18T18:37:17.863406Z","shell.execute_reply.started":"2022-11-18T18:37:13.490556Z","shell.execute_reply":"2022-11-18T18:37:17.862305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_concat.items():\n    if pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:33.169250Z","iopub.execute_input":"2022-11-18T18:37:33.169611Z","iopub.status.idle":"2022-11-18T18:37:33.551093Z","shell.execute_reply.started":"2022-11-18T18:37:33.169582Z","shell.execute_reply":"2022-11-18T18:37:33.550150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Filling and turning categorical variable into numbers\nfor label,content in df_concat.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:40.244912Z","iopub.execute_input":"2022-11-18T18:37:40.245485Z","iopub.status.idle":"2022-11-18T18:37:40.253242Z","shell.execute_reply.started":"2022-11-18T18:37:40.245436Z","shell.execute_reply":"2022-11-18T18:37:40.252153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_concat.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        #Add binary column to indicate whether sample had missing value\n        \n        # Turn categories into numbers and add +1\n        df_concat[label] =pd.Categorical(content).codes + 1","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:51.878045Z","iopub.execute_input":"2022-11-18T18:37:51.878462Z","iopub.status.idle":"2022-11-18T18:37:51.884789Z","shell.execute_reply.started":"2022-11-18T18:37:51.878423Z","shell.execute_reply":"2022-11-18T18:37:51.883793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label,content in df_concat.items():\n    if not pd.api.types.is_numeric_dtype(content):\n        if pd.isnull(content).sum():\n            print(label)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T18:37:59.663853Z","iopub.execute_input":"2022-11-18T18:37:59.664265Z","iopub.status.idle":"2022-11-18T18:37:59.670285Z","shell.execute_reply.started":"2022-11-18T18:37:59.664229Z","shell.execute_reply":"2022-11-18T18:37:59.669063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the model from disk\nimport pickle\nloaded_model = pickle.load(open('../input/finalized-model/finalized_model.sav', 'rb'))\nresult = pd.DataFrame(loaded_model.predict_proba(df_concat),columns = ['prob_0','prob_1'])\nfinal_result=pd.concat([df_concat_1,result],axis=1,ignore_index = False)\nfinal_result.set_index('customer_ID', inplace=True)\nfinal_result.to_csv('final.csv') \n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-18T19:00:42.493575Z","iopub.execute_input":"2022-11-18T19:00:42.493932Z","iopub.status.idle":"2022-11-18T19:01:46.170733Z","shell.execute_reply.started":"2022-11-18T19:00:42.493900Z","shell.execute_reply":"2022-11-18T19:01:46.169034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}