{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-01-16T05:59:35.571555Z","iopub.execute_input":"2023-01-16T05:59:35.572424Z","iopub.status.idle":"2023-01-16T05:59:35.580440Z","shell.execute_reply.started":"2023-01-16T05:59:35.572390Z","shell.execute_reply":"2023-01-16T05:59:35.579591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install scikit-learn","metadata":{"execution":{"iopub.status.busy":"2023-01-16T05:59:35.581907Z","iopub.execute_input":"2023-01-16T05:59:35.582256Z","iopub.status.idle":"2023-01-16T05:59:42.915246Z","shell.execute_reply.started":"2023-01-16T05:59:35.582232Z","shell.execute_reply":"2023-01-16T05:59:42.914314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install seaborn","metadata":{"execution":{"iopub.status.busy":"2023-01-16T05:59:42.916538Z","iopub.execute_input":"2023-01-16T05:59:42.916816Z","iopub.status.idle":"2023-01-16T05:59:47.155191Z","shell.execute_reply.started":"2023-01-16T05:59:42.916779Z","shell.execute_reply":"2023-01-16T05:59:47.154266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2023-01-16T05:59:47.157474Z","iopub.execute_input":"2023-01-16T05:59:47.157784Z","iopub.status.idle":"2023-01-16T05:59:53.274047Z","shell.execute_reply.started":"2023-01-16T05:59:47.157748Z","shell.execute_reply":"2023-01-16T05:59:53.273134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install lightgbm","metadata":{"execution":{"iopub.status.busy":"2023-01-16T05:59:53.275352Z","iopub.execute_input":"2023-01-16T05:59:53.275653Z","iopub.status.idle":"2023-01-16T05:59:58.103405Z","shell.execute_reply.started":"2023-01-16T05:59:53.275625Z","shell.execute_reply":"2023-01-16T05:59:58.102361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install xgboost","metadata":{"execution":{"iopub.status.busy":"2023-01-16T05:59:58.104881Z","iopub.execute_input":"2023-01-16T05:59:58.105204Z","iopub.status.idle":"2023-01-16T06:00:08.881864Z","shell.execute_reply.started":"2023-01-16T05:59:58.105171Z","shell.execute_reply":"2023-01-16T06:00:08.880741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:00:08.883222Z","iopub.execute_input":"2023-01-16T06:00:08.883550Z","iopub.status.idle":"2023-01-16T06:00:11.406426Z","shell.execute_reply.started":"2023-01-16T06:00:08.883514Z","shell.execute_reply":"2023-01-16T06:00:11.405132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I used AMEX-Feather-Dataset because the dataset that is provided in this competion is very large and reading raw CSV files,lead me to create a out of memory error. ","metadata":{}},{"cell_type":"code","source":"temp_train_dataset = pd.read_feather('../input/amexfeather/train_data.ftr')\n# Keep the latest statement features for each customer\ntrain_dataset = temp_train_dataset.groupby('customer_ID').tail(1).set_index('customer_ID', drop=True).sort_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:00:11.407996Z","iopub.execute_input":"2023-01-16T06:00:11.408460Z","iopub.status.idle":"2023-01-16T06:00:27.572193Z","shell.execute_reply.started":"2023-01-16T06:00:11.408425Z","shell.execute_reply":"2023-01-16T06:00:27.571003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_test_dataset = pd.read_feather('../input/amexfeather/test_data.ftr')\n# Keep the latest statement features for each customer\ntest_dataset = temp_test_dataset.groupby('customer_ID').tail(1).set_index('customer_ID', drop=True).sort_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:00:27.573576Z","iopub.execute_input":"2023-01-16T06:00:27.574078Z","iopub.status.idle":"2023-01-16T06:01:02.768912Z","shell.execute_reply.started":"2023-01-16T06:00:27.574047Z","shell.execute_reply":"2023-01-16T06:01:02.767754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del temp_train_dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:02.772796Z","iopub.execute_input":"2023-01-16T06:01:02.773115Z","iopub.status.idle":"2023-01-16T06:01:02.887497Z","shell.execute_reply.started":"2023-01-16T06:01:02.773086Z","shell.execute_reply":"2023-01-16T06:01:02.886546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del temp_test_dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:02.888624Z","iopub.execute_input":"2023-01-16T06:01:02.888898Z","iopub.status.idle":"2023-01-16T06:01:03.007120Z","shell.execute_reply.started":"2023-01-16T06:01:02.888874Z","shell.execute_reply":"2023-01-16T06:01:03.006140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:03.008223Z","iopub.execute_input":"2023-01-16T06:01:03.008508Z","iopub.status.idle":"2023-01-16T06:01:03.043700Z","shell.execute_reply.started":"2023-01-16T06:01:03.008484Z","shell.execute_reply":"2023-01-16T06:01:03.042848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:03.044822Z","iopub.execute_input":"2023-01-16T06:01:03.045145Z","iopub.status.idle":"2023-01-16T06:01:03.071990Z","shell.execute_reply.started":"2023-01-16T06:01:03.045116Z","shell.execute_reply":"2023-01-16T06:01:03.071138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.info(max_cols=191,show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:03.073076Z","iopub.execute_input":"2023-01-16T06:01:03.073366Z","iopub.status.idle":"2023-01-16T06:01:03.394973Z","shell.execute_reply.started":"2023-01-16T06:01:03.073341Z","shell.execute_reply":"2023-01-16T06:01:03.393911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:03.396410Z","iopub.execute_input":"2023-01-16T06:01:03.396771Z","iopub.status.idle":"2023-01-16T06:01:15.224339Z","shell.execute_reply.started":"2023-01-16T06:01:03.396738Z","shell.execute_reply":"2023-01-16T06:01:15.223407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset contains aggregated profile features for each customer at each statement date. Features are anonymized and normalized, and fall into five general categories:\n\n* D_* = Delinquency variables\n* S_* = Spend variables\n* P_* = Payment variables\n* B_* = Balance variables\n* R_* = Risk variables\n\nThe following features are categorical: ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{}},{"cell_type":"code","source":"categorical_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nnum_cols = [col for col in train_dataset.columns if col not in categorical_cols + [\"target\"]]\n\nprint(f'Total number of features: {1}')\nprint(f'Total number of categorical features: {len(categorical_cols)}')\nprint(f'Total number of continuos features: {len(num_cols)}')","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:15.225665Z","iopub.execute_input":"2023-01-16T06:01:15.226459Z","iopub.status.idle":"2023-01-16T06:01:15.232960Z","shell.execute_reply.started":"2023-01-16T06:01:15.226423Z","shell.execute_reply":"2023-01-16T06:01:15.232061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize the target variable","metadata":{}},{"cell_type":"code","source":"sns.countplot(x = 'target', data = train_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:15.234156Z","iopub.execute_input":"2023-01-16T06:01:15.234460Z","iopub.status.idle":"2023-01-16T06:01:15.598449Z","shell.execute_reply.started":"2023-01-16T06:01:15.234427Z","shell.execute_reply":"2023-01-16T06:01:15.597430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize categorial features","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 30))\nfor i, k in enumerate(categorical_cols):\n    plt.subplot(6, 2, i+1)\n    temp_val = pd.DataFrame(train_dataset[k].value_counts(dropna=False, normalize=True).sort_index().rename('count'))\n    temp_val.index.name = 'value'\n    temp_val.reset_index(inplace=True)\n    plt.bar(temp_val.index, temp_val['count'], alpha=0.5)\n    plt.xlabel(k)\n    plt.ylabel('frequency')\n    plt.xticks(temp_val.index, temp_val.value)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:15.599750Z","iopub.execute_input":"2023-01-16T06:01:15.600038Z","iopub.status.idle":"2023-01-16T06:01:17.007133Z","shell.execute_reply.started":"2023-01-16T06:01:15.600013Z","shell.execute_reply":"2023-01-16T06:01:17.006034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize aggregated profile features","metadata":{}},{"cell_type":"code","source":"Delinquency = [d for d in train_dataset.columns if d.startswith('D_')]\nSpend = [s for s in train_dataset.columns if s.startswith('S_')]\nPayment = [p for p in train_dataset.columns if p.startswith('P_')]\nBalance = [b for b in train_dataset.columns if b.startswith('B_')]\nRisk = [r for r in train_dataset.columns if r.startswith('R_')]\nDict = {'Delinquency': len(Delinquency), 'Spend': len(Spend), 'Payment': len(Payment), 'Balance': len(Balance), 'Risk': len(Risk),}\n\nplt.figure(figsize=(10,5))\nsns.barplot(x=list(Dict.keys()), y=list(Dict.values()));","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:17.008527Z","iopub.execute_input":"2023-01-16T06:01:17.008821Z","iopub.status.idle":"2023-01-16T06:01:17.172540Z","shell.execute_reply.started":"2023-01-16T06:01:17.008795Z","shell.execute_reply":"2023-01-16T06:01:17.171703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking for null values in test data set and train data set","metadata":{}},{"cell_type":"code","source":"NaN_Val = np.array(train_dataset.isnull().sum())\nNaN_prec = np.array((train_dataset.isnull().sum() * 100 / len(train_dataset)).round(2))\nNaN_Col = pd.DataFrame([np.array(list(train_dataset.columns)).T,NaN_Val.T,NaN_prec.T,np.array(list(train_dataset.dtypes)).T], index=['Features','Num of Missing values','Percentage','DataType']\n).transpose()\npd.set_option('display.max_rows', None)\nNaN_Col","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:17.173696Z","iopub.execute_input":"2023-01-16T06:01:17.173959Z","iopub.status.idle":"2023-01-16T06:01:17.767807Z","shell.execute_reply.started":"2023-01-16T06:01:17.173936Z","shell.execute_reply":"2023-01-16T06:01:17.766726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NaN_Val2 = np.array(test_dataset.isnull().sum())\nNaN_prec2 = np.array((test_dataset.isnull().sum() * 100 / len(test_dataset)).round(2))\nNaN_Col2 = pd.DataFrame([np.array(list(test_dataset.columns)).T,NaN_Val2.T,NaN_prec2.T,np.array(list(test_dataset.dtypes)).T], index=['Features','Num of Missing values','Percentage','DataType']\n).transpose()\npd.set_option('display.max_rows', None)\n\nNaN_Col2","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:17.769157Z","iopub.execute_input":"2023-01-16T06:01:17.769474Z","iopub.status.idle":"2023-01-16T06:01:18.860730Z","shell.execute_reply.started":"2023-01-16T06:01:17.769447Z","shell.execute_reply":"2023-01-16T06:01:18.859797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I rmoved columns if there are > 80% of missing values","metadata":{}},{"cell_type":"code","source":"train_dataset = train_dataset.drop(['S_2','D_66','D_42','D_49','D_73','D_76','R_9','B_29','D_87','D_88','D_106','R_26','D_108','D_110','D_111','B_39','B_42','D_132','D_134','D_135','D_136','D_137','D_138','D_142'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:18.861993Z","iopub.execute_input":"2023-01-16T06:01:18.862290Z","iopub.status.idle":"2023-01-16T06:01:19.005892Z","shell.execute_reply.started":"2023-01-16T06:01:18.862265Z","shell.execute_reply":"2023-01-16T06:01:19.004699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = test_dataset.drop(['S_2','D_42','D_49','D_66','D_73','D_76','R_9','B_29','D_87','D_88','D_106','R_26','D_108','D_110','D_111','B_39','B_42','D_132','D_134','D_135','D_136','D_137','D_138','D_142'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:19.007457Z","iopub.execute_input":"2023-01-16T06:01:19.007817Z","iopub.status.idle":"2023-01-16T06:01:19.276921Z","shell.execute_reply.started":"2023-01-16T06:01:19.007787Z","shell.execute_reply":"2023-01-16T06:01:19.275757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Filling null values with the median because mean value is not suitable as it returns Nan","metadata":{}},{"cell_type":"code","source":"selected_column = np.array(['P_2','S_3','B_2','D_41','D_43','B_3','D_44','D_45','D_46','D_48','D_50','D_53','S_7','D_56','S_9','B_6','B_8','D_52','P_3','D_54','D_55','B_13','D_59','D_61','B_15','D_62','B_16','B_17','D_77','B_19','B_20','D_69','B_22','D_70','D_72','D_74','R_7','B_25','B_26','D_78','D_79','D_80','B_27','D_81','R_12','D_82','D_105','S_27','D_83','R_14','D_84','D_86','R_20','B_33','D_89','D_91','S_22','S_23','S_24','S_25','S_26','D_102','D_103','D_104','D_107','B_37','R_27','D_109','D_112','B_40','D_113','D_115','D_118','D_119','D_121','D_122','D_123','D_124','D_125','D_128','D_129','B_41','D_130','D_131','D_133','D_139','D_140','D_141','D_143','D_144','D_145'])\n\nfor column in selected_column:\n    train_dataset[column] = train_dataset[column].fillna(train_dataset[column].median())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:19.278256Z","iopub.execute_input":"2023-01-16T06:01:19.278694Z","iopub.status.idle":"2023-01-16T06:01:24.087831Z","shell.execute_reply.started":"2023-01-16T06:01:19.278666Z","shell.execute_reply":"2023-01-16T06:01:24.086619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_column = np.array(['D_68','B_30','B_38','D_64','D_114','D_116','D_117','D_120','D_126'])\n\nfor column in selected_column:\n    train_dataset[column] =  train_dataset[column].fillna(train_dataset[column].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:24.089151Z","iopub.execute_input":"2023-01-16T06:01:24.089448Z","iopub.status.idle":"2023-01-16T06:01:24.133409Z","shell.execute_reply.started":"2023-01-16T06:01:24.089423Z","shell.execute_reply":"2023-01-16T06:01:24.132561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_column = np.array(['P_2','S_3','B_2','D_41','D_43','B_3','D_44','D_45','D_46','D_48','D_50','D_53','S_7','D_56','S_9','S_12','S_17','B_6','B_8','D_52','P_3','D_54','D_55','B_13','D_59','D_61','B_15','D_62','B_16','B_17','D_77','B_19','B_20','D_69','B_22','D_70','D_72','D_74','R_7','B_25','B_26','D_78','D_79','D_80','B_27','D_81','R_12','D_82','D_105','S_27','D_83','R_14','D_84','D_86','R_20','B_33','D_89','D_91','S_22','S_23','S_24','S_25','S_26','D_102','D_103','D_104','D_107','B_37','R_27','D_109','D_112','B_40','D_113','D_115','D_118','D_119','D_121','D_122','D_123','D_124','D_125','D_128','D_129','B_41','D_130','D_131','D_133','D_139','D_140','D_141','D_143','D_144','D_145'])\n\nfor column in selected_column:\n    test_dataset[column] = test_dataset[column].fillna(test_dataset[column].median())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:24.134642Z","iopub.execute_input":"2023-01-16T06:01:24.134915Z","iopub.status.idle":"2023-01-16T06:01:34.057873Z","shell.execute_reply.started":"2023-01-16T06:01:24.134890Z","shell.execute_reply":"2023-01-16T06:01:34.056679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_column = np.array(['D_68','B_30','B_38','D_114','D_116','D_117','D_120','D_126'])\n\nfor column in selected_column:\n    test_dataset[column] =  test_dataset[column].fillna(test_dataset[column].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.059332Z","iopub.execute_input":"2023-01-16T06:01:34.059704Z","iopub.status.idle":"2023-01-16T06:01:34.137204Z","shell.execute_reply.started":"2023-01-16T06:01:34.059674Z","shell.execute_reply":"2023-01-16T06:01:34.135908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking null values after removing","metadata":{}},{"cell_type":"code","source":"print(\"Train data set\",train_dataset.isnull().sum().to_string())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.144002Z","iopub.execute_input":"2023-01-16T06:01:34.144351Z","iopub.status.idle":"2023-01-16T06:01:34.393844Z","shell.execute_reply.started":"2023-01-16T06:01:34.144321Z","shell.execute_reply":"2023-01-16T06:01:34.392821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test data set\",test_dataset.isnull().sum().to_string())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.395166Z","iopub.execute_input":"2023-01-16T06:01:34.395482Z","iopub.status.idle":"2023-01-16T06:01:34.894569Z","shell.execute_reply.started":"2023-01-16T06:01:34.395454Z","shell.execute_reply":"2023-01-16T06:01:34.893475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.895947Z","iopub.execute_input":"2023-01-16T06:01:34.896254Z","iopub.status.idle":"2023-01-16T06:01:34.902598Z","shell.execute_reply.started":"2023-01-16T06:01:34.896228Z","shell.execute_reply":"2023-01-16T06:01:34.901671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.903788Z","iopub.execute_input":"2023-01-16T06:01:34.904081Z","iopub.status.idle":"2023-01-16T06:01:34.941330Z","shell.execute_reply.started":"2023-01-16T06:01:34.904055Z","shell.execute_reply":"2023-01-16T06:01:34.940403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.942470Z","iopub.execute_input":"2023-01-16T06:01:34.942751Z","iopub.status.idle":"2023-01-16T06:01:34.948577Z","shell.execute_reply.started":"2023-01-16T06:01:34.942727Z","shell.execute_reply":"2023-01-16T06:01:34.947594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.949753Z","iopub.execute_input":"2023-01-16T06:01:34.950023Z","iopub.status.idle":"2023-01-16T06:01:34.984282Z","shell.execute_reply.started":"2023-01-16T06:01:34.950000Z","shell.execute_reply":"2023-01-16T06:01:34.983370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting categorical variable to numericals because models prefer numericals","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder\n\nenc = OrdinalEncoder()\ncategorical_cols.remove('D_66')\n\ntrain_dataset[categorical_cols] = enc.fit_transform(train_dataset[categorical_cols])\ntest_dataset[categorical_cols] = enc.transform(test_dataset[categorical_cols])","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:34.985772Z","iopub.execute_input":"2023-01-16T06:01:34.986115Z","iopub.status.idle":"2023-01-16T06:01:36.856219Z","shell.execute_reply.started":"2023-01-16T06:01:34.986089Z","shell.execute_reply":"2023-01-16T06:01:36.854980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remove highly correlated features if there are > 90% of correlations","metadata":{}},{"cell_type":"code","source":"train_dataset_without_target = train_dataset.drop([\"target\"],axis=1)\n\ncor_matrix = train_dataset_without_target.corr()\ncol_core = set()\n\nfor i in range(len(cor_matrix.columns)):\n    for j in range(i):\n        if(cor_matrix.iloc[i, j] > 0.9):\n            col_name = cor_matrix.columns[i]\n            col_core.add(col_name)\ncol_core","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:01:36.857763Z","iopub.execute_input":"2023-01-16T06:01:36.858110Z","iopub.status.idle":"2023-01-16T06:02:07.421119Z","shell.execute_reply.started":"2023-01-16T06:01:36.858080Z","shell.execute_reply":"2023-01-16T06:02:07.420007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = train_dataset.drop(col_core, axis=1)\ntest_dataset = test_dataset.drop(col_core, axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:07.422519Z","iopub.execute_input":"2023-01-16T06:02:07.422844Z","iopub.status.idle":"2023-01-16T06:02:07.865169Z","shell.execute_reply.started":"2023-01-16T06:02:07.422816Z","shell.execute_reply":"2023-01-16T06:02:07.863894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:07.866567Z","iopub.execute_input":"2023-01-16T06:02:07.867173Z","iopub.status.idle":"2023-01-16T06:02:07.872991Z","shell.execute_reply.started":"2023-01-16T06:02:07.867143Z","shell.execute_reply":"2023-01-16T06:02:07.872103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_columns = [col for col in train_dataset.columns if col not in [\"target\"]]\n\nX = train_dataset[num_columns]\ny = train_dataset['target']\n\nprint(f\"X shape is = {X.shape}\" )\nprint(f\"Y shape is = {y.shape}\" )","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:07.874119Z","iopub.execute_input":"2023-01-16T06:02:07.874371Z","iopub.status.idle":"2023-01-16T06:02:08.037268Z","shell.execute_reply.started":"2023-01-16T06:02:07.874348Z","shell.execute_reply":"2023-01-16T06:02:08.036092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train,x_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n\nprint(f\"X_train shape is = {x_train.shape}\" )\nprint(f\"Y_train shape is = {y_train.shape}\" )\nprint(f\"X_test shape is = {x_test.shape}\" )\nprint(f\"Y_test shape is = {y_test.shape}\" )","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:08.038691Z","iopub.execute_input":"2023-01-16T06:02:08.039001Z","iopub.status.idle":"2023-01-16T06:02:09.013285Z","shell.execute_reply.started":"2023-01-16T06:02:08.038973Z","shell.execute_reply":"2023-01-16T06:02:09.012201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Standardize the X and test dataset becase some models worked better with these values","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler2 = StandardScaler()\nscaler2.fit(X.copy())\nX_scaled = scaler2.transform(X.copy())\n\nscaler2.fit(test_dataset.copy())\ntest_data_scaled = scaler2.transform(test_dataset.copy())\npd.DataFrame(test_data_scaled).head()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:09.014648Z","iopub.execute_input":"2023-01-16T06:02:09.014950Z","iopub.status.idle":"2023-01-16T06:02:13.909270Z","shell.execute_reply.started":"2023-01-16T06:02:09.014923Z","shell.execute_reply":"2023-01-16T06:02:13.908341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"K-nearest neighbors ","metadata":{}},{"cell_type":"code","source":"# from sklearn.neighbors import KNeighborsClassifier\n# from sklearn.metrics import mean_squared_error\n# from sklearn.metrics import mean_absolute_error\n\n# knn = KNeighborsClassifier(n_neighbors=130)\n  \n# knn.fit(X, y)\n\n# # Make predictions on the test set\n# predictions = knn.predict(test_dataset[num_columns])\n\n\n# sample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\n# output = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\n# output.to_csv('submission_180313L_KNN.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:13.910658Z","iopub.execute_input":"2023-01-16T06:02:13.911027Z","iopub.status.idle":"2023-01-16T06:02:13.915784Z","shell.execute_reply.started":"2023-01-16T06:02:13.910995Z","shell.execute_reply":"2023-01-16T06:02:13.914975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Decision Tree","metadata":{}},{"cell_type":"code","source":"# from sklearn.tree import DecisionTreeClassifier\n\n\n# # Initialize the decision tree classifier\n# Decision_Tree_model = DecisionTreeClassifier(max_depth=10, min_samples_leaf=5, random_state=42)\n\n# # Train the classifier on the training data\n# Decision_Tree_model.fit(X, y)\n\n# # Make predictions on the test set\n# predictions = Decision_Tree_model.predict(test_dataset[num_columns])\n\n\n# sample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\n# output = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\n# output.to_csv('submission_180313L_Decision_Tree.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:13.916878Z","iopub.execute_input":"2023-01-16T06:02:13.917685Z","iopub.status.idle":"2023-01-16T06:02:13.927280Z","shell.execute_reply.started":"2023-01-16T06:02:13.917658Z","shell.execute_reply":"2023-01-16T06:02:13.926397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Support vector machine","metadata":{}},{"cell_type":"code","source":"# from sklearn.svm import LinearSVC\n\n# SVM_model = LinearSVC(C=0.1).fit(X, y)\n\n# # Make predictions on the test set\n# predictions = SVM_model.predict(test_dataset[num_columns])\n\n\n# sample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\n# output = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\n# output.to_csv('submission_180313L_SVM.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:13.928249Z","iopub.execute_input":"2023-01-16T06:02:13.928898Z","iopub.status.idle":"2023-01-16T06:02:13.936096Z","shell.execute_reply.started":"2023-01-16T06:02:13.928872Z","shell.execute_reply":"2023-01-16T06:02:13.935281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Logistic Regression","metadata":{}},{"cell_type":"code","source":"# from sklearn.linear_model import LogisticRegression\n\n# LogisticRegression_model = LogisticRegression().fit(X, y)\n\n# predictions = LogisticRegression_model.predict(test_dataset[num_columns])\n\n\n# sample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\n# output = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\n# output.to_csv('submission_180313L_LogisticRegression.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:13.937120Z","iopub.execute_input":"2023-01-16T06:02:13.937392Z","iopub.status.idle":"2023-01-16T06:02:13.945897Z","shell.execute_reply.started":"2023-01-16T06:02:13.937356Z","shell.execute_reply":"2023-01-16T06:02:13.945062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LightGBM classifier","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\nd_train = lgb.Dataset(x_train, label=y_train, categorical_feature = categorical_cols)\n\nparams = {'objective': 'binary','n_estimators': 1200,'metric': 'binary_logloss','boosting': 'gbdt','num_leaves': 90,'reg_lambda' : 50,'colsample_bytree': 0.19,'learning_rate': 0.03,'min_child_samples': 2400,'max_bins': 511,'seed': 42,'verbose': -1}\n\n# trained model with 100 iterations\nlgb_model = lgb.train(params, d_train, 100)\n\npredictions = lgb_model.predict(test_dataset[num_columns])\n\nsample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\noutput = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\noutput.to_csv('submission_180313L_lightgbm.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:02:13.946920Z","iopub.execute_input":"2023-01-16T06:02:13.947176Z","iopub.status.idle":"2023-01-16T06:03:37.166404Z","shell.execute_reply.started":"2023-01-16T06:02:13.947154Z","shell.execute_reply":"2023-01-16T06:03:37.165354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBoost classifier","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\nd_train = xgb.DMatrix(X_scaled, label=y)\n\nparams = {'objective': 'binary:logistic','reg_lambda' : 50,'colsample_bytree': 0.19,'learning_rate': 0.03,'seed': 42}\n\n# trained model with 1000 iterations\nxgb_model = xgb.train(params, d_train, 1000)\n\nd_test = xgb.DMatrix(test_data_scaled)\npredictions = xgb_model.predict(d_test)\n\nsample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\noutput = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\noutput.to_csv('submission_180313L_xgboost.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:03:37.167749Z","iopub.execute_input":"2023-01-16T06:03:37.168117Z","iopub.status.idle":"2023-01-16T06:06:48.997716Z","shell.execute_reply.started":"2023-01-16T06:03:37.168083Z","shell.execute_reply":"2023-01-16T06:06:48.996580Z"},"trusted":true},"execution_count":null,"outputs":[]}]}