{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import roc_auc_score\n\nimport lightgbm as lgb\n\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-01T09:49:25.744615Z","iopub.execute_input":"2022-06-01T09:49:25.745943Z","iopub.status.idle":"2022-06-01T09:49:25.756786Z","shell.execute_reply.started":"2022-06-01T09:49:25.745865Z","shell.execute_reply":"2022-06-01T09:49:25.755049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hello kagglers\nI want to share with you one finding that i made by performing adversarial validation between test set and training set. \n*Refere to [this notebook](https://www.kaggle.com/code/carlmcbrideellis/what-is-adversarial-validation/notebook) if you want to understand what is adversarial validation and how to use that as a validation strategy.*\n\nMy guessing is that the **test set is shifted in time** respect to the training set and there are some features that contains a seasonal component like D_59, S_11, D_121, S_9.\n\nEDIT: i found that https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense comes to the same conclusion.","metadata":{}},{"cell_type":"code","source":"# Data took from processed https://www.kaggle.com/datasets/ruchi798/parquet-files-amexdefault-prediction\n\ntrain_data = pd.read_feather(\n    '../input/parquet-files-amexdefault-prediction/train_data.ftr'\n)\n\ntest_data = pd.read_feather(\n    '../input/parquet-files-amexdefault-prediction/test_data.ftr'\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:25.841629Z","iopub.execute_input":"2022-06-01T09:49:25.842143Z","iopub.status.idle":"2022-06-01T09:49:47.816729Z","shell.execute_reply.started":"2022-06-01T09:49:25.842103Z","shell.execute_reply":"2022-06-01T09:49:47.813515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code took from https://www.kaggle.com/code/lucasmorin/amex-lgbm-features-eng MANY THANKS!!\n\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    \n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n\n# from ambrosm notebook\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(pd.DataFrame({'target': y_true}), pd.Series(y_pred, name='prediction')),\n            True)\n\n#see : https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534\ndef amex_metric_mod_lgbm(y_pred: np.ndarray, data: lgb.Dataset):\n\n    y_true = data.get_label()\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 'AMEX', 0.5 * (gini[1]/gini[0]+ top_four), True   ","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:47.820461Z","iopub.execute_input":"2022-06-01T09:49:47.821583Z","iopub.status.idle":"2022-06-01T09:49:47.852031Z","shell.execute_reply.started":"2022-06-01T09:49:47.821539Z","shell.execute_reply":"2022-06-01T09:49:47.850733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = (\n    train_data\n    .groupby('customer_ID')\n    .tail(1)\n    .set_index('customer_ID', drop=True)\n    .sort_index()\n    .drop(['S_2'], axis='columns')\n)\n\ntrain_data['target'] = 0\ntrain_data.reset_index(inplace=True, drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:47.853892Z","iopub.execute_input":"2022-06-01T09:49:47.854437Z","iopub.status.idle":"2022-06-01T09:49:51.171908Z","shell.execute_reply.started":"2022-06-01T09:49:47.854392Z","shell.execute_reply":"2022-06-01T09:49:51.170841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = (\n    test_data\n    .groupby('customer_ID')\n    .tail(1)\n    .set_index('customer_ID', drop=True)\n    .sort_index()\n    .drop(['S_2'], axis='columns')\n)\n\ntest_data['target'] = 1\n\ntest_data.reset_index(inplace=True, drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:51.174199Z","iopub.execute_input":"2022-06-01T09:49:51.174658Z","iopub.status.idle":"2022-06-01T09:49:58.018758Z","shell.execute_reply.started":"2022-06-01T09:49:51.174627Z","shell.execute_reply":"2022-06-01T09:49:58.017425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_data.append(\n    test_data\n)\n\ndel test_data, train_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:58.020266Z","iopub.execute_input":"2022-06-01T09:49:58.020654Z","iopub.status.idle":"2022-06-01T09:49:59.624409Z","shell.execute_reply.started":"2022-06-01T09:49:58.020621Z","shell.execute_reply":"2022-06-01T09:49:59.623243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split in training and testing set\ntrain, test = train_test_split(df, test_size=0.2, random_state=1234)\ntrain.reset_index(drop=True, inplace=True)\ntest.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:49:59.626371Z","iopub.execute_input":"2022-06-01T09:49:59.627547Z","iopub.status.idle":"2022-06-01T09:50:04.601513Z","shell.execute_reply.started":"2022-06-01T09:49:59.627484Z","shell.execute_reply":"2022-06-01T09:50:04.600119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ohe encoding\ncat_featurs = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nenc = OneHotEncoder()\ntrain_enc_features = pd.DataFrame(\n    data=enc.fit_transform(train[cat_featurs]).toarray(),\n    columns=enc.get_feature_names_out()\n)\n\ntest_enc_features = pd.DataFrame(\n    data=enc.transform(test[cat_featurs]).toarray(),\n    columns=enc.get_feature_names_out()\n)\n\ntrain = pd.concat([train, train_enc_features], axis=1)\ntest = pd.concat([test, test_enc_features], axis=1)\n\ndel train_enc_features, test_enc_features\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:50:04.603065Z","iopub.execute_input":"2022-06-01T09:50:04.603461Z","iopub.status.idle":"2022-06-01T09:50:11.126327Z","shell.execute_reply.started":"2022-06-01T09:50:04.603427Z","shell.execute_reply":"2022-06-01T09:50:11.125030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [f for f in train.columns if f != 'target' and f != 'customer_ID' and f not in cat_featurs]\n\nparameters = {\n    'objective': 'binary',\n    'boosting': 'gbdt',\n    'learning_rate': 0.05,\n    'min_child_samples': 1000,\n    'reg_lambda':10,\n    'verbose': 25,\n    'seed':1234,\n    'n_estimators':100\n}\n\ntrain_df, eval_df = train_test_split(train, test_size=0.2, random_state=1234)\n\n\nlgb_train_data = lgb.Dataset(train_df[features], label=train_df['target'])\nlgb_test_data = lgb.Dataset(eval_df[features], label=eval_df['target'])\n\nclf = lgb.train(\n    parameters,\n    lgb_train_data,\n    valid_sets=lgb_test_data,\n    early_stopping_rounds=50\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:50:11.127670Z","iopub.execute_input":"2022-06-01T09:50:11.128105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = clf.predict(test[features], num_iteration=clf.best_iteration)\nscore = roc_auc_score(test['target'], preds)\n\ndel train_df, eval_df, lgb_test_data, lgb_train_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.DataFrame(\n    {\n        'feat_imp': clf.feature_importance(),\n        'feat_name': clf.feature_name()\n    }\n).sort_values(by='feat_imp', ascending=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp[:6]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, col in feat_imp.iloc[:6].iterrows():\n    plt.figure()\n    plt.title(\n        f\"distribution for {col['feat_name']}\"\n    )\n    plt.hist(\n        train.loc[train['target']==1, col['feat_name']],\n        color='orange',\n        alpha=0.3,\n        label = 'test_set',\n        bins=100,\n        density=True\n    )\n    plt.hist(\n        train.loc[train['target']==0, col['feat_name']],\n        color='blue',\n        alpha=0.3,\n        label = 'train_set',\n        bins=100,\n        density=True        \n    )\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['valid_prob'] = clf.predict(\n    train[features],\n    num_iteration=clf.best_iteration\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.hist(train['valid_prob'], bins=100)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_data = pd.read_feather(\n    '../input/parquet-files-amexdefault-prediction/train_data.ftr'\n)\n\ninf_data = (\n    inf_data\n    .groupby('customer_ID')\n    .tail(1)\n    .set_index('customer_ID', drop=True)\n    .sort_index()\n    .drop(['S_2'], axis='columns')\n)\n\ninf_enc_features = pd.DataFrame(\n    data=enc.transform(inf_data[cat_featurs]).toarray(),\n    columns=enc.get_feature_names_out()\n)\n\ninference = pd.concat([inf_data, inf_enc_features], axis=1)\n\ndel inf_data, inf_enc_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference['valid_prob'] = clf.predict(\n    inference[features],\n    num_iteration=clf.best_iteration\n)\n\ninference.reset_index(inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.hist(inference['valid_prob'], bins=100)\nplt.show()\n\ninference.sort_values(\n    by='valid_prob', \n    inplace=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference['valid_prob'].quantile(\n    q=0.9\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# infer over the training set for successive works\ninference[['index', 'valid_prob']].to_pickle(\n    './adversarial_df.pkl'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I hope that you find that intresting and you can use that in order to improve your score.\n\n# THANK YOU!!","metadata":{}}]}