{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom imblearn.over_sampling import SMOTE\nimport warnings, gc\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.preprocessing import LabelEncoder,MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,confusion_matrix\n\nimport lightgbm as lgb\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\n\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\n\nimport catboost as cb\nfrom catboost import CatBoostClassifier\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\nlabel = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntrain = train.merge(label,how='inner',on=\"customer_ID\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlab = LabelEncoder()\ntrain['customer_ID']= lab.fit_transform(train['customer_ID'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.groupby(['customer_ID']).tail(1).set_index('customer_ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['customer_ID']= lab.fit_transform(test['customer_ID'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.groupby('customer_ID').tail(1).set_index('customer_ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.target\nX = train.drop([\"target\",\"S_2\"],axis=1)\ntest = test.drop([\"S_2\"],axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.fillna(-123)\ntest = test.fillna(-123)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols =['B_30', 'B_38', 'D_63', 'D_64', 'D_66', 'D_68', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126']\nnum_cols = [col for col in X.columns if col not in cat_cols ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_n_cols = [col for col in num_cols if col.startswith(\"D\")]\nS_n_cols = [col for col in num_cols if col.startswith(\"S\")]\nP_n_cols = [col for col in num_cols if col.startswith(\"P\")]\nB_n_cols = [col for col in num_cols if col.startswith(\"B\")]\nR_n_cols = [col for col in num_cols if col.startswith(\"R\")]\nD_c_cols = [col for col in cat_cols if col.startswith(\"D\")]\nB_c_cols = [col for col in cat_cols if col.startswith(\"B\")] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nX_num_agg_D = X.groupby(\"customer_ID\")[D_n_cols].agg(['mean','min', 'last'])\nX_num_agg_D.columns = ['_'.join(x) for x in X_num_agg_D.columns]\n\nX_num_agg_S = X.groupby(\"customer_ID\")[S_n_cols].agg(['mean','min', 'last'])\nX_num_agg_S.columns = ['_'.join(x) for x in X_num_agg_S.columns]\n\nX_num_agg_P = X.groupby(\"customer_ID\")[P_n_cols].agg(['mean','min','max' ,'last'])\nX_num_agg_P.columns = ['_'.join(x) for x in X_num_agg_P.columns]\n\nX_num_agg_B = X.groupby(\"customer_ID\")[B_n_cols].agg(['mean','min', 'last'])\nX_num_agg_B.columns = ['_'.join(x) for x in X_num_agg_B.columns]\n\nX_num_agg_R = X.groupby(\"customer_ID\")[R_n_cols].agg(['mean','min','last'])\nX_num_agg_R.columns = ['_'.join(x) for x in X_num_agg_R.columns]\n\nX_cat_agg_D = X.groupby(\"customer_ID\")[D_c_cols].agg([ 'count','last','first','nunique'])\nX_cat_agg_D.columns = ['_'.join(x) for x in X_cat_agg_D.columns]\n\nX_cat_agg_B = X.groupby(\"customer_ID\")[B_c_cols].agg([ 'count','last','nunique'])\nX_cat_agg_B.columns = ['_'.join(x) for x in X_cat_agg_B.columns]\n\nX = pd.concat([X_num_agg_D, X_num_agg_S,X_num_agg_P,X_num_agg_B,X_num_agg_R,X_cat_agg_D,X_cat_agg_B], axis=1)\ndel X_num_agg_D, X_num_agg_S,X_num_agg_P,X_num_agg_B,X_num_agg_R,X_cat_agg_D,X_cat_agg_B\n_ = gc.collect()\n\nprint('X shape after engineering', X.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \ntest_num_agg_D = test.groupby(\"customer_ID\")[D_n_cols].agg(['mean','min', 'last'])\ntest_num_agg_D.columns = ['_'.join(x) for x in test_num_agg_D.columns]\n\ntest_num_agg_S = test.groupby(\"customer_ID\")[S_n_cols].agg(['mean','min', 'last'])\ntest_num_agg_S.columns = ['_'.join(x) for x in test_num_agg_S.columns]\n\ntest_num_agg_P = test.groupby(\"customer_ID\")[P_n_cols].agg(['mean','min','max', 'last'])\ntest_num_agg_P.columns = ['_'.join(x) for x in test_num_agg_P.columns]\n\ntest_num_agg_B = test.groupby(\"customer_ID\")[B_n_cols].agg(['mean','min', 'last'])\ntest_num_agg_B.columns = ['_'.join(x) for x in test_num_agg_B.columns]\n\ntest_num_agg_R = test.groupby(\"customer_ID\")[R_n_cols].agg(['mean','min', 'last'])\ntest_num_agg_R.columns = ['_'.join(x) for x in test_num_agg_R.columns]\n\ntest_cat_agg_D = test.groupby(\"customer_ID\")[D_c_cols].agg(['count','first', 'last','nunique'])\ntest_cat_agg_D.columns = ['_'.join(x) for x in test_cat_agg_D.columns]\n\ntest_cat_agg_B = test.groupby(\"customer_ID\")[B_c_cols].agg([ 'count','last','nunique'])\ntest_cat_agg_B.columns = ['_'.join(x) for x in test_cat_agg_B.columns]\n\ntest = pd.concat([test_num_agg_D, test_num_agg_S,test_num_agg_P,test_num_agg_B,test_num_agg_R,test_cat_agg_D,test_cat_agg_B], axis=1)\ndel test_num_agg_D, test_num_agg_S,test_num_agg_P,test_num_agg_B,test_num_agg_R,test_cat_agg_D,test_cat_agg_B\n_ = gc.collect()\n\nprint('Test shape after engineering', test.shape)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''%%time\nsmote = SMOTE(sampling_strategy='minority')\nX_sm,y_sm = smote.fit_resample(X,y)\ny_sm.value_counts()\n'''\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_cat=X[0:152971]\nX_xgbm =X[152971:305942]\nX_lgbm =X[305942:]\n\ny_cat=y[0:152971]\ny_xgbm =y[152971:305942]\ny_lgbm =y[305942:]\n\nX_cat.shape,y_cat.shape,X_xgbm.shape,y_xgbm.shape,X_lgbm.shape,y_lgbm.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBM","metadata":{}},{"cell_type":"code","source":"X_train_xgbm,X_valid_xgbm,y_train_xgbm,y_valid_xgbm = train_test_split(X_xgbm, y_xgbm, test_size=0.25,stratify=y_xgbm)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_parms ={\n    'booster': 'dart',\n     'n_jobs':4,\n     'n_estimators':500,\n    'lambda': 4.091409953463271e-08,\n    'alpha': 3.6353429991712695e-08,\n    'subsample': 0.6423675532438815,\n    'colsample_bytree': 0.7830450413657872,\n    'max_depth': 9,\n    'min_child_weight': 5,\n    'eta': 0.3749337530972536,\n    'gamma': 0.0745370910451703,\n    'grow_policy': 'depthwise',\n    'sample_type': 'uniform',\n    'normalize_type': 'tree',\n    'rate_drop': 0.0723975209176045,\n    'skip_drop': 0.9026367296518939}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgbm = XGBClassifier(**xgb_parms)\nxgbm.fit(X_train_xgbm, y_train_xgbm, \n             early_stopping_rounds=10, \n             eval_set=[(X_valid_xgbm, y_valid_xgbm)],\n             verbose=0)  \n                          ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprdeictions_xgbm = xgbm.predict_proba(test)\npreds_xgbm = pd.DataFrame(prdeictions_xgbm)\npred_final_xgbm = np.array(preds_xgbm[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Light_GBM","metadata":{}},{"cell_type":"code","source":"X_train_lgbm,X_valid_lgbm,y_train_lgbm,y_valid_lgbm = train_test_split(X_lgbm, y_lgbm, test_size=0.25,stratify=y_lgbm)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_lgbm = {\n            'metric': \"binary_logloss\",\n            'boosting_type': \"dart\",\n            'n_estimators':1000,\n            'verbosity': -1,\n            'lambda_l1': 3.1412416493672213e-06,\n             'lambda_l2': 1.919550703890871,\n             'num_leaves': 53,\n             'feature_fraction': 0.8361911823947347,\n             'bagging_fraction': 0.5246353885003125,\n             'bagging_freq': 7,\n             'min_child_samples': 91}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm =LGBMClassifier(**param_lgbm).fit(X_train_lgbm, y_train_lgbm, \n                                       eval_set=[(X_train_lgbm, y_train_lgbm), (X_valid_lgbm, y_valid_lgbm)],\n                                       \n                                       eval_metric=['auc','binary_logloss'],verbose=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprdeictions_lgbm = lgbm.predict_proba(test)\n\npreds_lgbm = pd.DataFrame(prdeictions_lgbm)\npred_final_lgbm = np.array(preds_lgbm[1])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost","metadata":{}},{"cell_type":"code","source":"X_train_cat,X_valid_cat,y_train_cat,y_valid_cat = train_test_split(X_cat, y_cat, test_size=0.25,stratify=y_cat)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Params_cat={ \n    'objective': 'CrossEntropy',\n    'n_estimators':1000,\n    'colsample_bylevel': 0.07868805912943484,\n    'depth': 9,\n    'boosting_type': 'Plain',\n    'bootstrap_type': 'MVS',\n    }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cbm =CatBoostClassifier(**Params_cat).fit(X_train_cat, y_train_cat, \n                                       eval_set=[(X_train_cat, y_train_cat), (X_valid_cat, y_valid_cat)],\n                                      verbose=0,\n                                       \n                                       )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprdeictions_cbm = cbm.predict_proba(test)\npreds_cbm = pd.DataFrame(prdeictions_cbm)\npred_final_cbm = np.array(preds_cbm[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npred_df= pd.DataFrame({'lgbm':pred_final_lgbm,'xgbm':pred_final_xgbm,'catboost':pred_final_cbm})\n\npred_df['mean'] = (pred_df['lgbm'] + pred_df['xgbm'] + pred_df['catboost'])/3\n\npred_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(pred_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/amex-default-prediction/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['prediction']=pred_df['mean'].values\n\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}