{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt, gc\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom catboost import CatBoostClassifier, Pool, EShapCalcType, EFeaturesSelectionAlgorithm\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!jupyter nbextension enable --py widgetsnbextension","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 22\n\nN_FOLDS = 5\n\nTEST_SIZE_SPLIT = 0.2\n\nNUM_FEATURES_TO_SELECT = 618    # Drop 300 feats from 918\n\nSTEPS_TO_SELECT = 3\n\nCATBOOST_PARAMS = dict(iterations=5000,\n                       learning_rate=0.067666,\n                       #depth=7,\n                       #l2_leaf_reg=40,\n                       #bootstrap_type='Bernoulli',    # bootstrap_type='Bayesian',\n                       #subsample=0.7,                 # bagging_temperature=3,\n                       #scale_pos_weight=3,\n                       #eval_metric='AUC',\n                       metric_period=100,\n                       task_type='GPU',\n                       od_type='Iter',\n                       od_wait=20,\n                       random_seed=SEED,\n                       allow_writing_files=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_pickle(\"../input/amex-agg-data-pickle/train_agg.pkl\", compression=\"gzip\")\ntrain.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\"\n]\ncat_features = [f\"{cf}_last\" for cf in cat_features]\nle_encoder = LabelEncoder()\nfor categorical_feature in cat_features:\n    train[categorical_feature] = le_encoder.fit_transform(train[categorical_feature])\n    \ntrain_y = pd.DataFrame(train[\"target\"])\ntrain_x = train.drop(\"target\", axis=1)\ntrain_cols = train_x.columns\ndel train, le_encoder\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n        \n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntr_x, val_x, tr_y, val_y = train_test_split(train_x, train_y, test_size=TEST_SIZE_SPLIT, random_state=SEED)\n\ndel train_x, train_y\n_ = gc.collect()\n\ntrain_pool = Pool(tr_x, tr_y, cat_features=cat_features)\nval_pool = Pool(val_x, val_y, cat_features=cat_features)\n\ndel tr_x, tr_y, val_x, val_y\n_ = gc.collect()\n\nclf = CatBoostClassifier(**CATBOOST_PARAMS)\n\nsummary = clf.select_features(\n    train_pool,\n    eval_set=val_pool,\n    features_for_select=train_cols.values.tolist(),\n    num_features_to_select=NUM_FEATURES_TO_SELECT,\n    steps=STEPS_TO_SELECT,\n    algorithm=EFeaturesSelectionAlgorithm.RecursiveByShapValues,\n    shap_calc_type=EShapCalcType.Regular,\n    train_final_model=False,\n    verbose=True,\n    plot=True\n)\n\n#print('Selected features names:', summary['selected_features_names'])\ncat_features = [c for c in cat_features if c not in summary['eliminated_features_names']]\ndel train_pool, val_pool, clf\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_pickle(\"../input/amex-agg-data-pickle/train_agg.pkl\", compression=\"gzip\")\ntest = pd.read_pickle(\"../input/amex-agg-data-pickle/test_agg.pkl\", compression=\"gzip\")\n\nle_encoder = LabelEncoder()\nfor categorical_feature in cat_features:\n    train[categorical_feature] = le_encoder.fit_transform(train[categorical_feature])\n    test[categorical_feature] = le_encoder.transform(test[categorical_feature])\n    \ntest = test[summary['selected_features_names']]\ndel le_encoder\n_ = gc.collect()\n\ntrain_y = pd.DataFrame(train[\"target\"])\ntrain_x = train.drop(\"target\", axis=1)\ndel train\n_ = gc.collect()\n\ntrain_x = train_x[summary['selected_features_names']]\ndel summary\n_ = gc.collect()\n\ntrain_cols = train_x.columns\nprint(train_x.shape)\nprint(test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importances = []\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\ny_oof = np.zeros(train_x.shape[0])\ny_test = np.zeros(test.shape[0])\nix = 0\nfor train_ind, val_ind in skf.split(train_x, train_y):\n    print(f\"******* Fold {ix} ******* \")\n    tr_x, val_x = (\n        train_x.iloc[train_ind].reset_index(drop=True),\n        train_x.iloc[val_ind].reset_index(drop=True),\n    )\n    tr_y, val_y = (\n        train_y.iloc[train_ind].reset_index(drop=True),\n        train_y.iloc[val_ind].reset_index(drop=True),\n    )\n    \n    clf = CatBoostClassifier(**CATBOOST_PARAMS)\n    \n    clf.fit(tr_x, tr_y, eval_set=[(val_x, val_y)], cat_features=cat_features, use_best_model=True,\n                                                                                     verbose=True)\n    \n    fea_imp = pd.DataFrame({'col': train_cols, f'imp_{ix}': clf.feature_importances_})\n    importances.append(fea_imp)\n    \n    preds = clf.predict_proba(val_x)[:, 1]\n    y_oof[val_ind] = y_oof[val_ind] + preds\n    \n    val_score_fold = amex_metric_mod(val_y.values.flatten(), preds)\n    print('Kaggle Metric =',val_score_fold,'\\n')\n    \n    preds_test = clf.predict_proba(test)[:, 1]\n    y_test = y_test + preds_test / N_FOLDS\n    ix = ix + 1\n    \n    del tr_x, val_x, tr_y, val_y, clf, fea_imp, preds, preds_test\n    _ = gc.collect()\n    \ny_pred = train_y.copy(deep=True)\ny_pred = y_pred.rename(columns={\"target\": \"prediction\"})\ny_pred[\"prediction\"] = y_oof\nval_score = amex_metric_mod(train_y.values.flatten(), y_pred.values.flatten())\nprint(f\"OVERALL CV Amex metric: {val_score}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_FEATURES = 20\n\ndf_imp = importances[0].copy()\nfor k in range(1,N_FOLDS): df_imp = df_imp.merge(importances[k], on='col', how='left')\ndf_imp['importance'] = df_imp.iloc[:,1:].mean(axis=1)\ndf_imp = df_imp.sort_values('importance',ascending=False)\n\nplt.figure(figsize=(10,5*NUM_FEATURES//10))\nplt.barh(np.arange(NUM_FEATURES,0,-1), df_imp['importance'].values[:NUM_FEATURES])\nplt.yticks(np.arange(NUM_FEATURES,0,-1), df_imp['col'].values[:NUM_FEATURES])\nplt.title(f'CatBoost Feature Importance - Top {NUM_FEATURES}')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"prediction\"] = y_test\ntest[\"prediction\"].to_csv(f\"submission_cat_{val_score}.csv\", index=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reference\nhttps://www.kaggle.com/code/huseyincot/amex-catboost-0-793 CV 0.79056 LB 0.793\n\nhttps://github.com/catboost/tutorials/blob/master/feature_selection/select_features_tutorial.ipynb","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}