{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-28T15:31:28.458940Z","iopub.execute_input":"2022-06-28T15:31:28.459997Z","iopub.status.idle":"2022-06-28T15:31:28.473443Z","shell.execute_reply.started":"2022-06-28T15:31:28.459943Z","shell.execute_reply":"2022-06-28T15:31:28.472114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About the Dataset\nThe dataset contains aggregated profile features for each customer at each statement date. Features are anonymized and normalized, and fall into the following general categories:\n\nD_* = Delinquency variables<br>\nS_* = Spend variables<br>\nP_* = Payment variables<br>\nB_* = Balance variables<br>\nR_* = Risk variables<br>\n\nwith the following features being categorical:\n\n['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nMy task is to predict, for each customer_ID, the probability of a future payment default (target = 1)\n\n**Note that the negative class has been subsampled for this dataset at 5%, and thus receives a 20x weighting in the scoring metric.**","metadata":{}},{"cell_type":"markdown","source":"# 1. Importing the libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgbm\nfrom lightgbm import LGBMClassifier\nimport os\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport gc \nwarnings.filterwarnings('ignore')\npd.set_option(\"display.max_columns\", None)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:31:28.576551Z","iopub.execute_input":"2022-06-28T15:31:28.576826Z","iopub.status.idle":"2022-06-28T15:31:31.590490Z","shell.execute_reply.started":"2022-06-28T15:31:28.576800Z","shell.execute_reply":"2022-06-28T15:31:31.589518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet').groupby('customer_ID').tail(2).set_index('customer_ID', drop=True).sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:31:31.592435Z","iopub.execute_input":"2022-06-28T15:31:31.593114Z","iopub.status.idle":"2022-06-28T15:31:55.906716Z","shell.execute_reply.started":"2022-06-28T15:31:31.593074Z","shell.execute_reply":"2022-06-28T15:31:55.905544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target=pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:31:55.908422Z","iopub.execute_input":"2022-06-28T15:31:55.908825Z","iopub.status.idle":"2022-06-28T15:31:56.823162Z","shell.execute_reply.started":"2022-06-28T15:31:55.908783Z","shell.execute_reply":"2022-06-28T15:31:56.821887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:31:56.826093Z","iopub.execute_input":"2022-06-28T15:31:56.826801Z","iopub.status.idle":"2022-06-28T15:31:56.846971Z","shell.execute_reply.started":"2022-06-28T15:31:56.826760Z","shell.execute_reply":"2022-06-28T15:31:56.845877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.merge(train_df, train_target, on='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:31:56.848459Z","iopub.execute_input":"2022-06-28T15:31:56.849027Z","iopub.status.idle":"2022-06-28T15:32:07.630917Z","shell.execute_reply.started":"2022-06-28T15:31:56.848989Z","shell.execute_reply":"2022-06-28T15:32:07.629059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:07.632805Z","iopub.execute_input":"2022-06-28T15:32:07.633423Z","iopub.status.idle":"2022-06-28T15:32:07.837793Z","shell.execute_reply.started":"2022-06-28T15:32:07.633381Z","shell.execute_reply":"2022-06-28T15:32:07.836808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:07.839434Z","iopub.execute_input":"2022-06-28T15:32:07.840031Z","iopub.status.idle":"2022-06-28T15:32:07.847562Z","shell.execute_reply.started":"2022-06-28T15:32:07.839976Z","shell.execute_reply":"2022-06-28T15:32:07.846167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:07.849511Z","iopub.execute_input":"2022-06-28T15:32:07.850356Z","iopub.status.idle":"2022-06-28T15:32:08.014554Z","shell.execute_reply.started":"2022-06-28T15:32:07.850317Z","shell.execute_reply":"2022-06-28T15:32:08.013005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:08.016015Z","iopub.execute_input":"2022-06-28T15:32:08.016637Z","iopub.status.idle":"2022-06-28T15:32:08.045069Z","shell.execute_reply.started":"2022-06-28T15:32:08.016597Z","shell.execute_reply":"2022-06-28T15:32:08.044127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:08.050242Z","iopub.execute_input":"2022-06-28T15:32:08.050835Z","iopub.status.idle":"2022-06-28T15:32:12.712805Z","shell.execute_reply.started":"2022-06-28T15:32:08.050794Z","shell.execute_reply":"2022-06-28T15:32:12.711820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat=['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:12.713864Z","iopub.execute_input":"2022-06-28T15:32:12.714226Z","iopub.status.idle":"2022-06-28T15:32:12.720697Z","shell.execute_reply.started":"2022-06-28T15:32:12.714175Z","shell.execute_reply":"2022-06-28T15:32:12.719179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data Cleaning","metadata":{}},{"cell_type":"code","source":"# Missing values\ntmp = train_df.isna().sum().mul(100).div(len(train_df)).sort_values(ascending=False)\ntmp[:15]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:12.722516Z","iopub.execute_input":"2022-06-28T15:32:12.723420Z","iopub.status.idle":"2022-06-28T15:32:13.125875Z","shell.execute_reply.started":"2022-06-28T15:32:12.723378Z","shell.execute_reply":"2022-06-28T15:32:13.124756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dropping columns with missing values >70%\nmissingDF = pd.DataFrame(tmp).reset_index()\ndrop_cols = missingDF[missingDF[0]>70][\"index\"].values\nprint(drop_cols)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.127534Z","iopub.execute_input":"2022-06-28T15:32:13.127926Z","iopub.status.idle":"2022-06-28T15:32:13.136670Z","shell.execute_reply.started":"2022-06-28T15:32:13.127886Z","shell.execute_reply":"2022-06-28T15:32:13.135520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dropping Null values >70%","metadata":{}},{"cell_type":"code","source":"train_df.drop(columns = drop_cols,axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.139033Z","iopub.execute_input":"2022-06-28T15:32:13.139456Z","iopub.status.idle":"2022-06-28T15:32:13.331332Z","shell.execute_reply.started":"2022-06-28T15:32:13.139424Z","shell.execute_reply":"2022-06-28T15:32:13.330260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.332668Z","iopub.execute_input":"2022-06-28T15:32:13.333087Z","iopub.status.idle":"2022-06-28T15:32:13.432943Z","shell.execute_reply.started":"2022-06-28T15:32:13.333046Z","shell.execute_reply":"2022-06-28T15:32:13.432024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.434151Z","iopub.execute_input":"2022-06-28T15:32:13.434902Z","iopub.status.idle":"2022-06-28T15:32:13.443617Z","shell.execute_reply.started":"2022-06-28T15:32:13.434863Z","shell.execute_reply":"2022-06-28T15:32:13.442538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For categorical columns\ncols = train_df.columns\nnum_cols = train_df._get_numeric_data().columns","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.444812Z","iopub.execute_input":"2022-06-28T15:32:13.446122Z","iopub.status.idle":"2022-06-28T15:32:13.452518Z","shell.execute_reply.started":"2022-06-28T15:32:13.446022Z","shell.execute_reply":"2022-06-28T15:32:13.451329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.454734Z","iopub.execute_input":"2022-06-28T15:32:13.455330Z","iopub.status.idle":"2022-06-28T15:32:13.465154Z","shell.execute_reply.started":"2022-06-28T15:32:13.455279Z","shell.execute_reply":"2022-06-28T15:32:13.464238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.466171Z","iopub.execute_input":"2022-06-28T15:32:13.469464Z","iopub.status.idle":"2022-06-28T15:32:13.479378Z","shell.execute_reply.started":"2022-06-28T15:32:13.469425Z","shell.execute_reply":"2022-06-28T15:32:13.478239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.480974Z","iopub.execute_input":"2022-06-28T15:32:13.481751Z","iopub.status.idle":"2022-06-28T15:32:13.489865Z","shell.execute_reply.started":"2022-06-28T15:32:13.481703Z","shell.execute_reply":"2022-06-28T15:32:13.488620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns = list(set(cols) - set(cat))\nfiltered_numerical_columns = list(set(train_df[numerical_columns])-{\"S_2\",\"customer_ID\"})","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.490957Z","iopub.execute_input":"2022-06-28T15:32:13.493769Z","iopub.status.idle":"2022-06-28T15:32:13.730823Z","shell.execute_reply.started":"2022-06-28T15:32:13.493730Z","shell.execute_reply":"2022-06-28T15:32:13.729777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(filtered_numerical_columns)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.732586Z","iopub.execute_input":"2022-06-28T15:32:13.733278Z","iopub.status.idle":"2022-06-28T15:32:13.742438Z","shell.execute_reply.started":"2022-06-28T15:32:13.733237Z","shell.execute_reply":"2022-06-28T15:32:13.740521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat:\n    print(i + \" Attribute is  of Data Type : \"+ str(train_df[i].dtypes))","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.744975Z","iopub.execute_input":"2022-06-28T15:32:13.746503Z","iopub.status.idle":"2022-06-28T15:32:13.760253Z","shell.execute_reply.started":"2022-06-28T15:32:13.746462Z","shell.execute_reply":"2022-06-28T15:32:13.758914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat:\n    train_df[i] = train_df[i].astype(\"object\")\n    print(i + \" Attribute is  of Data Type : \"+ str(train_df[i].dtypes))","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:13.763592Z","iopub.execute_input":"2022-06-28T15:32:13.764111Z","iopub.status.idle":"2022-06-28T15:32:14.399650Z","shell.execute_reply.started":"2022-06-28T15:32:13.764083Z","shell.execute_reply":"2022-06-28T15:32:14.398597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[cat].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:14.400894Z","iopub.execute_input":"2022-06-28T15:32:14.401503Z","iopub.status.idle":"2022-06-28T15:32:15.459149Z","shell.execute_reply.started":"2022-06-28T15:32:14.401465Z","shell.execute_reply":"2022-06-28T15:32:15.458180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dirtiness in categorical data\nfor col in cat:\n    print('{} has {} values'.format(col,train_df[col].unique()))\n    print(\"\\n\")\n","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:15.461119Z","iopub.execute_input":"2022-06-28T15:32:15.461749Z","iopub.status.idle":"2022-06-28T15:32:16.039661Z","shell.execute_reply.started":"2022-06-28T15:32:15.461701Z","shell.execute_reply":"2022-06-28T15:32:16.038619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cat)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:16.041170Z","iopub.execute_input":"2022-06-28T15:32:16.041568Z","iopub.status.idle":"2022-06-28T15:32:16.048745Z","shell.execute_reply.started":"2022-06-28T15:32:16.041528Z","shell.execute_reply":"2022-06-28T15:32:16.047339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,20))\n\nfor i,feature in enumerate(cat):\n    plt.subplot(4,3,i+1)\n    sns.countplot(train_df[feature])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:16.061436Z","iopub.execute_input":"2022-06-28T15:32:16.062075Z","iopub.status.idle":"2022-06-28T15:32:43.299089Z","shell.execute_reply.started":"2022-06-28T15:32:16.062045Z","shell.execute_reply":"2022-06-28T15:32:43.298113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"target\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:43.300643Z","iopub.execute_input":"2022-06-28T15:32:43.301358Z","iopub.status.idle":"2022-06-28T15:32:43.315317Z","shell.execute_reply.started":"2022-06-28T15:32:43.301313Z","shell.execute_reply":"2022-06-28T15:32:43.314258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,20))\n\nfor i,feature in enumerate(cat):\n    plt.subplot(4,3,i+1)\n    sns.countplot(train_df[feature],hue=train_df['target'])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:32:43.316712Z","iopub.execute_input":"2022-06-28T15:32:43.317628Z","iopub.status.idle":"2022-06-28T15:33:12.956073Z","shell.execute_reply.started":"2022-06-28T15:32:43.317587Z","shell.execute_reply":"2022-06-28T15:33:12.955112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(train_df['target'])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:12.957805Z","iopub.execute_input":"2022-06-28T15:33:12.958506Z","iopub.status.idle":"2022-06-28T15:33:13.152063Z","shell.execute_reply.started":"2022-06-28T15:33:12.958463Z","shell.execute_reply":"2022-06-28T15:33:13.150938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Correlation between features","metadata":{}},{"cell_type":"code","source":"# For numeric columns filling null values\nfiltered_numerical_columns = train_df.select_dtypes(np.number).columns\ntrain_df[filtered_numerical_columns] = train_df[filtered_numerical_columns].fillna(train_df[filtered_numerical_columns].mean())","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:13.157649Z","iopub.execute_input":"2022-06-28T15:33:13.158174Z","iopub.status.idle":"2022-06-28T15:33:14.590642Z","shell.execute_reply.started":"2022-06-28T15:33:13.158122Z","shell.execute_reply":"2022-06-28T15:33:14.589507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:14.592387Z","iopub.execute_input":"2022-06-28T15:33:14.592756Z","iopub.status.idle":"2022-06-28T15:33:15.859906Z","shell.execute_reply.started":"2022-06-28T15:33:14.592715Z","shell.execute_reply":"2022-06-28T15:33:15.858970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in filtered_numerical_columns:\n    print(i + \" Attribute is  of Data Type : \"+ str(train_df[i].dtypes))","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:15.861535Z","iopub.execute_input":"2022-06-28T15:33:15.861912Z","iopub.status.idle":"2022-06-28T15:33:15.879862Z","shell.execute_reply.started":"2022-06-28T15:33:15.861874Z","shell.execute_reply":"2022-06-28T15:33:15.878878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[filtered_numerical_columns][:5]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:15.881094Z","iopub.execute_input":"2022-06-28T15:33:15.881547Z","iopub.status.idle":"2022-06-28T15:33:16.132840Z","shell.execute_reply.started":"2022-06-28T15:33:15.881507Z","shell.execute_reply":"2022-06-28T15:33:16.131576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Performing the Feature Encoding\nMachine learning models can only work with numerical values. For this reason, it is necessary to transform the categorical values of the relevant features into numerical ones. This process is called feature encoding.","metadata":{}},{"cell_type":"code","source":"for col in cat:\n    print('{} has {} categories'.format(col,train_df[col].nunique()))","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:16.134463Z","iopub.execute_input":"2022-06-28T15:33:16.135087Z","iopub.status.idle":"2022-06-28T15:33:16.723228Z","shell.execute_reply.started":"2022-06-28T15:33:16.135045Z","shell.execute_reply":"2022-06-28T15:33:16.722256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['S_2'] = pd.to_datetime(train_df['S_2'], errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:16.724872Z","iopub.execute_input":"2022-06-28T15:33:16.725284Z","iopub.status.idle":"2022-06-28T15:33:16.989981Z","shell.execute_reply.started":"2022-06-28T15:33:16.725244Z","shell.execute_reply":"2022-06-28T15:33:16.988944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling date column\n\ntrain_df[\"S_2_day\"] =train_df[\"S_2\"].dt.day\ntrain_df[\"S_2_month\"] = train_df[\"S_2\"].dt.month\ntrain_df[\"S_2_year\"] = train_df[\"S_2\"].dt.year","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:16.991812Z","iopub.execute_input":"2022-06-28T15:33:16.992303Z","iopub.status.idle":"2022-06-28T15:33:17.206108Z","shell.execute_reply.started":"2022-06-28T15:33:16.992255Z","shell.execute_reply":"2022-06-28T15:33:17.205102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop S_2\ntrain_df.drop(columns=[\"S_2\"], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:17.207623Z","iopub.execute_input":"2022-06-28T15:33:17.207994Z","iopub.status.idle":"2022-06-28T15:33:17.503959Z","shell.execute_reply.started":"2022-06-28T15:33:17.207954Z","shell.execute_reply":"2022-06-28T15:33:17.502864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['customer_ID'].head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:17.505488Z","iopub.execute_input":"2022-06-28T15:33:17.506359Z","iopub.status.idle":"2022-06-28T15:33:17.515080Z","shell.execute_reply.started":"2022-06-28T15:33:17.506317Z","shell.execute_reply":"2022-06-28T15:33:17.514128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# handling Cusotmer ID as it has unique data\ntrain_df =train_df.groupby(['customer_ID']).nth(-1).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:17.516486Z","iopub.execute_input":"2022-06-28T15:33:17.517092Z","iopub.status.idle":"2022-06-28T15:33:18.949447Z","shell.execute_reply.started":"2022-06-28T15:33:17.517034Z","shell.execute_reply":"2022-06-28T15:33:18.948292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:18.954223Z","iopub.execute_input":"2022-06-28T15:33:18.955091Z","iopub.status.idle":"2022-06-28T15:33:19.117921Z","shell.execute_reply.started":"2022-06-28T15:33:18.955052Z","shell.execute_reply":"2022-06-28T15:33:19.116814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label Encoding ---> Because there are less no. of categories in each column\nLabelEncoder can be used to normalize labels. It can also be used to transform non-numerical labels (as long as they are hashable and comparable) to numerical labels. Fit label encoder.\n","metadata":{}},{"cell_type":"code","source":" from sklearn.preprocessing import LabelEncoder","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:19.122181Z","iopub.execute_input":"2022-06-28T15:33:19.122770Z","iopub.status.idle":"2022-06-28T15:33:19.132332Z","shell.execute_reply.started":"2022-06-28T15:33:19.122730Z","shell.execute_reply":"2022-06-28T15:33:19.130925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le=LabelEncoder()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:19.137508Z","iopub.execute_input":"2022-06-28T15:33:19.137856Z","iopub.status.idle":"2022-06-28T15:33:19.142146Z","shell.execute_reply.started":"2022-06-28T15:33:19.137821Z","shell.execute_reply":"2022-06-28T15:33:19.141099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat:\n    train_df[col]=le.fit_transform(train_df[col])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:19.143840Z","iopub.execute_input":"2022-06-28T15:33:19.144244Z","iopub.status.idle":"2022-06-28T15:33:20.766285Z","shell.execute_reply.started":"2022-06-28T15:33:19.144183Z","shell.execute_reply":"2022-06-28T15:33:20.765250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:20.767757Z","iopub.execute_input":"2022-06-28T15:33:20.768394Z","iopub.status.idle":"2022-06-28T15:33:20.865796Z","shell.execute_reply.started":"2022-06-28T15:33:20.768352Z","shell.execute_reply":"2022-06-28T15:33:20.864863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Selecting important features\nSelectKBest: Feature selection is a technique where we choose those features in our data that contribute most to the target variable. In other words we choose the best predictors for the target variable. The classes in the sklearn.\n\nchi2: A chi-square (χ2) statistic is a test that measures how a model compares to actual observed data. ... The chi-square statistic compares the size any discrepancies between the expected results and the actual results, given the size of the sample and the number of variables in the relationship.","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:20.867236Z","iopub.execute_input":"2022-06-28T15:33:20.867835Z","iopub.status.idle":"2022-06-28T15:33:20.882084Z","shell.execute_reply.started":"2022-06-28T15:33:20.867796Z","shell.execute_reply":"2022-06-28T15:33:20.881241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import chi2,f_regression,mutual_info_classif","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:20.883830Z","iopub.execute_input":"2022-06-28T15:33:20.884427Z","iopub.status.idle":"2022-06-28T15:33:20.888838Z","shell.execute_reply.started":"2022-06-28T15:33:20.884387Z","shell.execute_reply":"2022-06-28T15:33:20.887838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind_col=[col for col in train_df.columns if col!='target']\ndep_col='target'","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:20.890489Z","iopub.execute_input":"2022-06-28T15:33:20.891189Z","iopub.status.idle":"2022-06-28T15:33:20.897502Z","shell.execute_reply.started":"2022-06-28T15:33:20.891124Z","shell.execute_reply":"2022-06-28T15:33:20.896456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As customer ID is not a usefull metric we will not use it for our model building","metadata":{}},{"cell_type":"code","source":"X=train_df[ind_col]\ny=train_df[dep_col]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:20.899383Z","iopub.execute_input":"2022-06-28T15:33:20.900372Z","iopub.status.idle":"2022-06-28T15:33:21.027112Z","shell.execute_reply.started":"2022-06-28T15:33:20.900332Z","shell.execute_reply":"2022-06-28T15:33:21.026058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:21.028712Z","iopub.execute_input":"2022-06-28T15:33:21.029365Z","iopub.status.idle":"2022-06-28T15:33:21.129553Z","shell.execute_reply.started":"2022-06-28T15:33:21.029321Z","shell.execute_reply":"2022-06-28T15:33:21.128651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:21.131090Z","iopub.execute_input":"2022-06-28T15:33:21.131465Z","iopub.status.idle":"2022-06-28T15:33:21.137504Z","shell.execute_reply.started":"2022-06-28T15:33:21.131428Z","shell.execute_reply":"2022-06-28T15:33:21.136519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_features=SelectKBest(score_func=mutual_info_classif,k=100)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:21.139173Z","iopub.execute_input":"2022-06-28T15:33:21.139883Z","iopub.status.idle":"2022-06-28T15:33:21.144594Z","shell.execute_reply.started":"2022-06-28T15:33:21.139844Z","shell.execute_reply":"2022-06-28T15:33:21.143656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_features=imp_features.fit(X,y)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:33:21.146175Z","iopub.execute_input":"2022-06-28T15:33:21.146822Z","iopub.status.idle":"2022-06-28T15:41:06.158541Z","shell.execute_reply.started":"2022-06-28T15:33:21.146784Z","shell.execute_reply":"2022-06-28T15:41:06.157447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_features","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.160423Z","iopub.execute_input":"2022-06-28T15:41:06.160818Z","iopub.status.idle":"2022-06-28T15:41:06.170366Z","shell.execute_reply.started":"2022-06-28T15:41:06.160776Z","shell.execute_reply":"2022-06-28T15:41:06.168991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_features.scores_","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.172102Z","iopub.execute_input":"2022-06-28T15:41:06.172835Z","iopub.status.idle":"2022-06-28T15:41:06.184075Z","shell.execute_reply.started":"2022-06-28T15:41:06.172798Z","shell.execute_reply":"2022-06-28T15:41:06.182695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datascore=pd.DataFrame(imp_features.scores_,columns=['Score'])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.185641Z","iopub.execute_input":"2022-06-28T15:41:06.186435Z","iopub.status.idle":"2022-06-28T15:41:06.191949Z","shell.execute_reply.started":"2022-06-28T15:41:06.186396Z","shell.execute_reply":"2022-06-28T15:41:06.190809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datascore","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.195407Z","iopub.execute_input":"2022-06-28T15:41:06.196226Z","iopub.status.idle":"2022-06-28T15:41:06.209585Z","shell.execute_reply.started":"2022-06-28T15:41:06.196158Z","shell.execute_reply":"2022-06-28T15:41:06.208551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.columns","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.211023Z","iopub.execute_input":"2022-06-28T15:41:06.211474Z","iopub.status.idle":"2022-06-28T15:41:06.219429Z","shell.execute_reply.started":"2022-06-28T15:41:06.211435Z","shell.execute_reply":"2022-06-28T15:41:06.218086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfcols=pd.DataFrame(X.columns)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.221104Z","iopub.execute_input":"2022-06-28T15:41:06.221636Z","iopub.status.idle":"2022-06-28T15:41:06.227549Z","shell.execute_reply.started":"2022-06-28T15:41:06.221597Z","shell.execute_reply":"2022-06-28T15:41:06.226054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfcols\n","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.229297Z","iopub.execute_input":"2022-06-28T15:41:06.230461Z","iopub.status.idle":"2022-06-28T15:41:06.241512Z","shell.execute_reply.started":"2022-06-28T15:41:06.230418Z","shell.execute_reply":"2022-06-28T15:41:06.240493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_rank=pd.concat([dfcols,datascore],axis=1)\nfeatures_rank","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.242966Z","iopub.execute_input":"2022-06-28T15:41:06.243777Z","iopub.status.idle":"2022-06-28T15:41:06.257394Z","shell.execute_reply.started":"2022-06-28T15:41:06.243750Z","shell.execute_reply":"2022-06-28T15:41:06.256454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_rank.columns=['features','score']","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.258588Z","iopub.execute_input":"2022-06-28T15:41:06.259372Z","iopub.status.idle":"2022-06-28T15:41:06.263754Z","shell.execute_reply.started":"2022-06-28T15:41:06.259342Z","shell.execute_reply":"2022-06-28T15:41:06.262695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_rank","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.265311Z","iopub.execute_input":"2022-06-28T15:41:06.266473Z","iopub.status.idle":"2022-06-28T15:41:06.280340Z","shell.execute_reply.started":"2022-06-28T15:41:06.266432Z","shell.execute_reply":"2022-06-28T15:41:06.279346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_rank.nlargest(100,'score')","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.281949Z","iopub.execute_input":"2022-06-28T15:41:06.282644Z","iopub.status.idle":"2022-06-28T15:41:06.297518Z","shell.execute_reply.started":"2022-06-28T15:41:06.282607Z","shell.execute_reply":"2022-06-28T15:41:06.296553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected=features_rank.nlargest(100,'score')['features'].values","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.299149Z","iopub.execute_input":"2022-06-28T15:41:06.299942Z","iopub.status.idle":"2022-06-28T15:41:06.306485Z","shell.execute_reply.started":"2022-06-28T15:41:06.299896Z","shell.execute_reply":"2022-06-28T15:41:06.305438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.308660Z","iopub.execute_input":"2022-06-28T15:41:06.310438Z","iopub.status.idle":"2022-06-28T15:41:06.317342Z","shell.execute_reply.started":"2022-06-28T15:41:06.310398Z","shell.execute_reply":"2022-06-28T15:41:06.315864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_new=X[selected]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.318933Z","iopub.execute_input":"2022-06-28T15:41:06.319684Z","iopub.status.idle":"2022-06-28T15:41:06.382594Z","shell.execute_reply.started":"2022-06-28T15:41:06.319647Z","shell.execute_reply":"2022-06-28T15:41:06.381557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = X_new.columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.384108Z","iopub.execute_input":"2022-06-28T15:41:06.384941Z","iopub.status.idle":"2022-06-28T15:41:06.389999Z","shell.execute_reply.started":"2022-06-28T15:41:06.384899Z","shell.execute_reply":"2022-06-28T15:41:06.388702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_new[cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.391519Z","iopub.execute_input":"2022-06-28T15:41:06.392806Z","iopub.status.idle":"2022-06-28T15:41:06.549047Z","shell.execute_reply.started":"2022-06-28T15:41:06.392746Z","shell.execute_reply":"2022-06-28T15:41:06.548087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_new.shape,y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.550577Z","iopub.execute_input":"2022-06-28T15:41:06.551186Z","iopub.status.idle":"2022-06-28T15:41:06.556746Z","shell.execute_reply.started":"2022-06-28T15:41:06.551143Z","shell.execute_reply":"2022-06-28T15:41:06.555758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_new.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.558732Z","iopub.execute_input":"2022-06-28T15:41:06.559369Z","iopub.status.idle":"2022-06-28T15:41:06.757804Z","shell.execute_reply.started":"2022-06-28T15:41:06.559328Z","shell.execute_reply":"2022-06-28T15:41:06.756377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.765860Z","iopub.execute_input":"2022-06-28T15:41:06.766521Z","iopub.status.idle":"2022-06-28T15:41:06.771904Z","shell.execute_reply.started":"2022-06-28T15:41:06.766477Z","shell.execute_reply":"2022-06-28T15:41:06.770741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_test,y_train,y_test=train_test_split(X_new,y,random_state=0,test_size=0.3)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:06.773745Z","iopub.execute_input":"2022-06-28T15:41:06.774243Z","iopub.status.idle":"2022-06-28T15:41:07.155903Z","shell.execute_reply.started":"2022-06-28T15:41:06.774182Z","shell.execute_reply":"2022-06-28T15:41:07.154756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.157685Z","iopub.execute_input":"2022-06-28T15:41:07.158068Z","iopub.status.idle":"2022-06-28T15:41:07.166406Z","shell.execute_reply.started":"2022-06-28T15:41:07.158021Z","shell.execute_reply":"2022-06-28T15:41:07.164882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts() ","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.168889Z","iopub.execute_input":"2022-06-28T15:41:07.169986Z","iopub.status.idle":"2022-06-28T15:41:07.182071Z","shell.execute_reply.started":"2022-06-28T15:41:07.169940Z","shell.execute_reply":"2022-06-28T15:41:07.180791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost Classifier - For our Model\nXGBoost is an optimized distributed gradient boosting library designed to be highly efficient, flexible and portable. It implements Machine Learning algorithms under the Gradient Boosting framework. It provides a parallel tree boosting to solve many data science problems in a fast and accurate way.\n\n# Since we are using XGBoost , feature scaling is not required","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBClassifier","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.183804Z","iopub.execute_input":"2022-06-28T15:41:07.184179Z","iopub.status.idle":"2022-06-28T15:41:07.274407Z","shell.execute_reply.started":"2022-06-28T15:41:07.184138Z","shell.execute_reply":"2022-06-28T15:41:07.273538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params={'learning-rate':[0,0.5,0.20,0.25],\n        'max_depth':[5,8,10],\n       'min_child_weight':[1,3,5,7],\n       'gamma':[0.0,0.1,0.2,0.4],\n       'colsample_bytree':[0.3,0.4,0.7]}","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.277131Z","iopub.execute_input":"2022-06-28T15:41:07.277491Z","iopub.status.idle":"2022-06-28T15:41:07.285031Z","shell.execute_reply.started":"2022-06-28T15:41:07.277461Z","shell.execute_reply":"2022-06-28T15:41:07.283848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.287169Z","iopub.execute_input":"2022-06-28T15:41:07.288090Z","iopub.status.idle":"2022-06-28T15:41:07.292457Z","shell.execute_reply.started":"2022-06-28T15:41:07.288042Z","shell.execute_reply":"2022-06-28T15:41:07.291372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier=XGBClassifier()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.294547Z","iopub.execute_input":"2022-06-28T15:41:07.294951Z","iopub.status.idle":"2022-06-28T15:41:07.302658Z","shell.execute_reply.started":"2022-06-28T15:41:07.294912Z","shell.execute_reply":"2022-06-28T15:41:07.301716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_search=RandomizedSearchCV(classifier,param_distributions=params,n_iter=5,scoring='roc_auc',n_jobs=-1,cv=5,verbose=3)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.304064Z","iopub.execute_input":"2022-06-28T15:41:07.305229Z","iopub.status.idle":"2022-06-28T15:41:07.311101Z","shell.execute_reply.started":"2022-06-28T15:41:07.305165Z","shell.execute_reply":"2022-06-28T15:41:07.309993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_search.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:41:07.324977Z","iopub.execute_input":"2022-06-28T15:41:07.325770Z","iopub.status.idle":"2022-06-28T16:40:58.127085Z","shell.execute_reply.started":"2022-06-28T15:41:07.325740Z","shell.execute_reply":"2022-06-28T16:40:58.125980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_search.best_estimator_ ","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:40:58.129045Z","iopub.execute_input":"2022-06-28T16:40:58.129717Z","iopub.status.idle":"2022-06-28T16:40:58.140640Z","shell.execute_reply.started":"2022-06-28T16:40:58.129672Z","shell.execute_reply":"2022-06-28T16:40:58.139408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:40:58.142530Z","iopub.execute_input":"2022-06-28T16:40:58.142877Z","iopub.status.idle":"2022-06-28T16:40:58.151677Z","shell.execute_reply.started":"2022-06-28T16:40:58.142840Z","shell.execute_reply":"2022-06-28T16:40:58.150352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n              colsample_bynode=1, colsample_bytree=0.3, gamma=0.2, gpu_id=-1,\n              importance_type='gain', interaction_constraints='', learning_rate=0.300000012, max_delta_step=0,\n              max_depth=5, min_child_weight=1,\n              monotone_constraints='()', n_estimators=100, n_jobs=8,\n              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n              scale_pos_weight=1, subsample=1, tree_method='exact',\n              validate_parameters=1, verbosity=None)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:40:58.153276Z","iopub.execute_input":"2022-06-28T16:40:58.153789Z","iopub.status.idle":"2022-06-28T16:40:58.162271Z","shell.execute_reply.started":"2022-06-28T16:40:58.153750Z","shell.execute_reply":"2022-06-28T16:40:58.161252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:40:58.163856Z","iopub.execute_input":"2022-06-28T16:40:58.164342Z","iopub.status.idle":"2022-06-28T16:42:35.976247Z","shell.execute_reply.started":"2022-06-28T16:40:58.164306Z","shell.execute_reply":"2022-06-28T16:42:35.975269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/inversion/amex-competition-metric-python\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:35.977715Z","iopub.execute_input":"2022-06-28T16:42:35.978320Z","iopub.status.idle":"2022-06-28T16:42:35.993120Z","shell.execute_reply.started":"2022-06-28T16:42:35.978280Z","shell.execute_reply":"2022-06-28T16:42:35.992131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's Predict our model Accuracy.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,accuracy_score\nfrom sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:35.994493Z","iopub.execute_input":"2022-06-28T16:42:35.994903Z","iopub.status.idle":"2022-06-28T16:42:36.004429Z","shell.execute_reply.started":"2022-06-28T16:42:35.994862Z","shell.execute_reply":"2022-06-28T16:42:36.003151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.006084Z","iopub.execute_input":"2022-06-28T16:42:36.006472Z","iopub.status.idle":"2022-06-28T16:42:36.376469Z","shell.execute_reply.started":"2022-06-28T16:42:36.006432Z","shell.execute_reply":"2022-06-28T16:42:36.375408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_prob = classifier.predict_proba(X_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.378455Z","iopub.execute_input":"2022-06-28T16:42:36.379168Z","iopub.status.idle":"2022-06-28T16:42:36.776420Z","shell.execute_reply.started":"2022-06-28T16:42:36.379126Z","shell.execute_reply":"2022-06-28T16:42:36.775379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.DataFrame(y_test, columns=[\"target\"])\ny_pred = pd.DataFrame(y_pred, columns=[\"prediction\"])\ny_pred_prob = pd.DataFrame(y_pred_prob, columns=[\"prediction\"])","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.778082Z","iopub.execute_input":"2022-06-28T16:42:36.778455Z","iopub.status.idle":"2022-06-28T16:42:36.790229Z","shell.execute_reply.started":"2022-06-28T16:42:36.778414Z","shell.execute_reply":"2022-06-28T16:42:36.788904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('MAE:',metrics.mean_absolute_error(y_test,y_pred))\nprint('MSE:',metrics.mean_squared_error(y_test,y_pred))\nprint(\"RMSE:\",np.sqrt(metrics.mean_squared_error(y_test,y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.791679Z","iopub.execute_input":"2022-06-28T16:42:36.792250Z","iopub.status.idle":"2022-06-28T16:42:36.811117Z","shell.execute_reply.started":"2022-06-28T16:42:36.792195Z","shell.execute_reply":"2022-06-28T16:42:36.810248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.812625Z","iopub.execute_input":"2022-06-28T16:42:36.813073Z","iopub.status.idle":"2022-06-28T16:42:36.844566Z","shell.execute_reply.started":"2022-06-28T16:42:36.813002Z","shell.execute_reply":"2022-06-28T16:42:36.843422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()\nprint(tn, fp, fn, tp)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.846385Z","iopub.execute_input":"2022-06-28T16:42:36.846746Z","iopub.status.idle":"2022-06-28T16:42:36.875346Z","shell.execute_reply.started":"2022-06-28T16:42:36.846709Z","shell.execute_reply":"2022-06-28T16:42:36.874379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.876682Z","iopub.execute_input":"2022-06-28T16:42:36.877239Z","iopub.status.idle":"2022-06-28T16:42:36.895982Z","shell.execute_reply.started":"2022-06-28T16:42:36.877175Z","shell.execute_reply":"2022-06-28T16:42:36.894960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"precision=tp/(tp+fp)\nrecall=tp/(tp+fn)\nprint(\"Precision : \",precision)\nprint(\"Recall : \",recall)\nf1score=(2*precision*recall)/(precision+recall)\nprint(\"F1 score: \",f1score)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:42:36.897679Z","iopub.execute_input":"2022-06-28T16:42:36.898027Z","iopub.status.idle":"2022-06-28T16:42:36.904783Z","shell.execute_reply.started":"2022-06-28T16:42:36.897991Z","shell.execute_reply":"2022-06-28T16:42:36.903470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  As the dataset is **imbalanced** accuracy can not be taken as a metric \n### F1 Score is our Model Metric # We Got Very Good F1 Score Using XGBoost : 79.9%<br>","metadata":{}},{"cell_type":"markdown","source":"# Official Metric ","metadata":{}},{"cell_type":"code","source":"# # computing metric score\namex_metric(y_test, y_pred_prob)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T15:29:35.997362Z","iopub.execute_input":"2022-06-28T15:29:35.998248Z","iopub.status.idle":"2022-06-28T15:29:36.474255Z","shell.execute_reply.started":"2022-06-28T15:29:35.998174Z","shell.execute_reply":"2022-06-28T15:29:36.473269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Results","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet').groupby('customer_ID').tail(2).set_index('customer_ID', drop=True).sort_index()\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:44:12.120311Z","iopub.execute_input":"2022-06-28T16:44:12.121063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test=test_df[selected]\nfinal_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel_test=[col for col in final_test.columns if col!='customer_ID']\nlen(sel_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final=final_test.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final=df_final.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:40:04.489096Z","iopub.execute_input":"2022-06-27T20:40:04.489513Z","iopub.status.idle":"2022-06-27T20:40:04.805461Z","shell.execute_reply.started":"2022-06-27T20:40:04.489479Z","shell.execute_reply":"2022-06-27T20:40:04.8043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:40:13.444535Z","iopub.execute_input":"2022-06-27T20:40:13.44548Z","iopub.status.idle":"2022-06-27T20:40:13.507147Z","shell.execute_reply.started":"2022-06-27T20:40:13.445443Z","shell.execute_reply":"2022-06-27T20:40:13.506034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.drop_duplicates(\"customer_ID\",inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:41:29.897069Z","iopub.execute_input":"2022-06-27T20:41:29.898215Z","iopub.status.idle":"2022-06-27T20:41:30.67618Z","shell.execute_reply.started":"2022-06-27T20:41:29.898161Z","shell.execute_reply":"2022-06-27T20:41:30.675076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:41:37.093072Z","iopub.execute_input":"2022-06-27T20:41:37.094301Z","iopub.status.idle":"2022-06-27T20:41:37.103401Z","shell.execute_reply.started":"2022-06-27T20:41:37.094249Z","shell.execute_reply":"2022-06-27T20:41:37.102385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data=df_final[sel_test]\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:41:43.181746Z","iopub.execute_input":"2022-06-27T20:41:43.182694Z","iopub.status.idle":"2022-06-27T20:41:43.36365Z","shell.execute_reply.started":"2022-06-27T20:41:43.182654Z","shell.execute_reply":"2022-06-27T20:41:43.362562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final[\"prediction\"]=xgb_classifier.predict(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:41:46.565454Z","iopub.execute_input":"2022-06-27T20:41:46.566223Z","iopub.status.idle":"2022-06-27T20:41:50.31014Z","shell.execute_reply.started":"2022-06-27T20:41:46.566182Z","shell.execute_reply":"2022-06-27T20:41:50.309042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.drop(selected,inplace=True,axis=1)\ndf_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:41:51.793248Z","iopub.execute_input":"2022-06-27T20:41:51.794329Z","iopub.status.idle":"2022-06-27T20:41:51.847921Z","shell.execute_reply.started":"2022-06-27T20:41:51.794273Z","shell.execute_reply":"2022-06-27T20:41:51.846981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final=df_final.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:42:18.180209Z","iopub.execute_input":"2022-06-27T20:42:18.180988Z","iopub.status.idle":"2022-06-27T20:42:18.202438Z","shell.execute_reply.started":"2022-06-27T20:42:18.180944Z","shell.execute_reply":"2022-06-27T20:42:18.20122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.drop(\"index\",inplace=True,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:42:50.750405Z","iopub.execute_input":"2022-06-27T20:42:50.750847Z","iopub.status.idle":"2022-06-27T20:42:50.790852Z","shell.execute_reply.started":"2022-06-27T20:42:50.750806Z","shell.execute_reply":"2022-06-27T20:42:50.789809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:42:52.688125Z","iopub.execute_input":"2022-06-27T20:42:52.688766Z","iopub.status.idle":"2022-06-27T20:42:52.699275Z","shell.execute_reply.started":"2022-06-27T20:42:52.688726Z","shell.execute_reply":"2022-06-27T20:42:52.69805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:43:07.39866Z","iopub.execute_input":"2022-06-27T20:43:07.399717Z","iopub.status.idle":"2022-06-27T20:43:07.407665Z","shell.execute_reply.started":"2022-06-27T20:43:07.399659Z","shell.execute_reply":"2022-06-27T20:43:07.406292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:43:10.111751Z","iopub.execute_input":"2022-06-27T20:43:10.112116Z","iopub.status.idle":"2022-06-27T20:43:10.121848Z","shell.execute_reply.started":"2022-06-27T20:43:10.112086Z","shell.execute_reply":"2022-06-27T20:43:10.120883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.index.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:43:14.161483Z","iopub.execute_input":"2022-06-27T20:43:14.162259Z","iopub.status.idle":"2022-06-27T20:43:14.173584Z","shell.execute_reply.started":"2022-06-27T20:43:14.162218Z","shell.execute_reply":"2022-06-27T20:43:14.172419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:43:28.77728Z","iopub.execute_input":"2022-06-27T20:43:28.778337Z","iopub.status.idle":"2022-06-27T20:43:28.785529Z","shell.execute_reply.started":"2022-06-27T20:43:28.778285Z","shell.execute_reply":"2022-06-27T20:43:28.784208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.to_csv(\"Submission_v4.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T20:44:02.293442Z","iopub.execute_input":"2022-06-27T20:44:02.29453Z","iopub.status.idle":"2022-06-27T20:44:05.411895Z","shell.execute_reply.started":"2022-06-27T20:44:02.294486Z","shell.execute_reply":"2022-06-27T20:44:05.410842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving The Model","metadata":{}},{"cell_type":"code","source":"\nimport joblib\njoblib.dump(classifier, \"xgboost_classifier_v2.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:43:39.531318Z","iopub.execute_input":"2022-06-28T16:43:39.532397Z","iopub.status.idle":"2022-06-28T16:43:39.545319Z","shell.execute_reply.started":"2022-06-28T16:43:39.532355Z","shell.execute_reply":"2022-06-28T16:43:39.543887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the model\nimport joblib\nxgb_classifier = joblib.load(\"./xgboost_classifier_v2.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:43:58.041345Z","iopub.execute_input":"2022-06-28T16:43:58.042349Z","iopub.status.idle":"2022-06-28T16:43:58.056407Z","shell.execute_reply.started":"2022-06-28T16:43:58.042306Z","shell.execute_reply":"2022-06-28T16:43:58.055443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred1=xgb_classifier.predict(X_test)\nmetrics.r2_score(y_test,y_pred1)","metadata":{"execution":{"iopub.status.busy":"2022-06-28T16:44:01.266770Z","iopub.execute_input":"2022-06-28T16:44:01.267415Z","iopub.status.idle":"2022-06-28T16:44:01.634233Z","shell.execute_reply.started":"2022-06-28T16:44:01.267375Z","shell.execute_reply":"2022-06-28T16:44:01.633216Z"},"trusted":true},"execution_count":null,"outputs":[]}]}