{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\nHere I will be using the pytorch implementation of [TabNet](https://github.com/dreamquark-ai/tabnet), which is an Attentive Interpretable Tabular Learning model. It has served me well in other competitions so I am also giving it a try on this one. It trains on GPU 😀\n\nTabNet paper can be found [here](https://arxiv.org/pdf/1908.07442.pdf).\n\nOn this example I am not using aggregated features.\n\n","metadata":{}},{"cell_type":"markdown","source":"# Installs/Imports","metadata":{}},{"cell_type":"code","source":"! pip install pytorch-tabnet -q","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:01.601846Z","iopub.execute_input":"2022-06-17T07:11:01.602295Z","iopub.status.idle":"2022-06-17T07:11:13.403664Z","shell.execute_reply.started":"2022-06-17T07:11:01.602207Z","shell.execute_reply":"2022-06-17T07:11:13.402681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport time\nimport psutil\nimport gc\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport torch\nfrom pytorch_tabnet.tab_model import TabNetClassifier\nfrom pytorch_tabnet.metrics import Metric","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:13.406218Z","iopub.execute_input":"2022-06-17T07:11:13.406887Z","iopub.status.idle":"2022-06-17T07:11:15.663543Z","shell.execute_reply.started":"2022-06-17T07:11:13.406843Z","shell.execute_reply":"2022-06-17T07:11:15.662332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\ntrain['S_2'] = pd.to_datetime(train['S_2']).astype('datetime64[ns]')\n\ntrain = train.groupby('customer_ID').tail(1).reset_index(drop=True)\ntrain = train.fillna(-1)\n\nprint(train.shape)\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:15.671409Z","iopub.execute_input":"2022-06-17T07:11:15.674809Z","iopub.status.idle":"2022-06-17T07:11:46.249755Z","shell.execute_reply.started":"2022-06-17T07:11:15.674763Z","shell.execute_reply":"2022-06-17T07:11:46.248769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n  DEBUG = True\n  model = 'tabnet'\n  N_folds = 5\n  seed = 42","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:46.251951Z","iopub.execute_input":"2022-06-17T07:11:46.253734Z","iopub.status.idle":"2022-06-17T07:11:46.259155Z","shell.execute_reply.started":"2022-06-17T07:11:46.253689Z","shell.execute_reply":"2022-06-17T07:11:46.258056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\nseed_everything(seed = CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:46.260621Z","iopub.execute_input":"2022-06-17T07:11:46.261022Z","iopub.status.idle":"2022-06-17T07:11:46.271253Z","shell.execute_reply.started":"2022-06-17T07:11:46.260994Z","shell.execute_reply":"2022-06-17T07:11:46.270315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"psutil.virtual_memory().percent","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:46.272479Z","iopub.execute_input":"2022-06-17T07:11:46.272888Z","iopub.status.idle":"2022-06-17T07:11:46.282235Z","shell.execute_reply.started":"2022-06-17T07:11:46.272845Z","shell.execute_reply":"2022-06-17T07:11:46.281180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Features and Target","metadata":{}},{"cell_type":"code","source":"target = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\nprint('target shape: ', target.shape)\n\ntrain = train.merge(target, on = 'customer_ID')\nprint('train shape: ', train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:46.283694Z","iopub.execute_input":"2022-06-17T07:11:46.284470Z","iopub.status.idle":"2022-06-17T07:11:54.952099Z","shell.execute_reply.started":"2022-06-17T07:11:46.284412Z","shell.execute_reply":"2022-06-17T07:11:54.951163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_features = [col for col in train.columns if col not in ['target', 'customer_ID', 'S_2']]\nn_features = len(all_features)\nprint('n features: ', n_features)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:54.953511Z","iopub.execute_input":"2022-06-17T07:11:54.954089Z","iopub.status.idle":"2022-06-17T07:11:54.961079Z","shell.execute_reply.started":"2022-06-17T07:11:54.954050Z","shell.execute_reply":"2022-06-17T07:11:54.960149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical features","metadata":{}},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:54.962590Z","iopub.execute_input":"2022-06-17T07:11:54.963036Z","iopub.status.idle":"2022-06-17T07:11:54.970194Z","shell.execute_reply.started":"2022-06-17T07:11:54.962995Z","shell.execute_reply":"2022-06-17T07:11:54.969432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_index = []\nfor cat in range(len(cat_features)):\n  cat_index.append(train.columns.get_loc(cat_features[cat]))\n\nprint(len(cat_index))","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:54.974117Z","iopub.execute_input":"2022-06-17T07:11:54.974462Z","iopub.status.idle":"2022-06-17T07:11:54.981639Z","shell.execute_reply.started":"2022-06-17T07:11:54.974411Z","shell.execute_reply":"2022-06-17T07:11:54.980551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric\n\nI will use [Vopani's implementation](https://www.kaggle.com/code/rohanrao/amex-competition-metric-implementations).","metadata":{}},{"cell_type":"code","source":"def amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:54.983468Z","iopub.execute_input":"2022-06-17T07:11:54.984356Z","iopub.status.idle":"2022-06-17T07:11:54.993638Z","shell.execute_reply.started":"2022-06-17T07:11:54.984298Z","shell.execute_reply":"2022-06-17T07:11:54.992821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# using amex metric to evaluate tabnet\nclass Amex_tabnet(Metric):\n    \n  def __init__(self):\n    self._name = 'amex_tabnet'\n    self._maximize = True\n\n  def __call__(self, y_true, y_pred):\n    amex = amex_metric_numpy(y_true, y_pred[:, 1])\n    return max(amex, 0.)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:54.995099Z","iopub.execute_input":"2022-06-17T07:11:54.995618Z","iopub.status.idle":"2022-06-17T07:11:55.004331Z","shell.execute_reply.started":"2022-06-17T07:11:54.995582Z","shell.execute_reply":"2022-06-17T07:11:55.003462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def run_training(X = train[all_features], y = train['target'],\n                 nfolds = CFG.N_folds):\n\n    print('\\n ', '-'*50)\n    print('\\nTraining: ', CFG.model)\n    print('\\n ', '-'*50)\n\n    print('\\nSeed: ', CFG.seed)\n    print('N folds: ', CFG.N_folds)\n    print('train shape: ', X.shape)\n    print('targets shape: ', y.shape)\n\n\n    print('\\nN features: ', len(all_features))\n    print('\\n')\n\n    models = list()\n    \n    kfold = StratifiedKFold(n_splits = CFG.N_folds, shuffle=True, random_state = CFG.seed)\n\n    for k, (train_idx, valid_idx) in enumerate(kfold.split(X, y)):\n\n        ## DEBUG MODE\n        if CFG.DEBUG == True:\n            if k > 0:\n                print('\\nDEBUG mode activated: Will train only one fold...\\n')\n                break      \n\n        start = time.time()\n\n        X_train, y_train = X.loc[train_idx], y.loc[train_idx]\n        X_valid, y_valid = X.loc[valid_idx], y.loc[valid_idx]        \n        \n        model = TabNetClassifier(n_d = 32,\n                                 n_a = 64,\n                                 n_steps = 3,\n                                 gamma = 1.3,\n                                 cat_idxs = cat_index,                                 \n                                 n_independent = 2,\n                                 n_shared = 2,\n                                 momentum = 0.02,\n                                 clip_value = None,\n                                 lambda_sparse = 1e-3,\n                                 optimizer_fn = torch.optim.Adam,\n                                 scheduler_fn = torch.optim.lr_scheduler.CosineAnnealingLR,\n                                 scheduler_params = {\"T_max\" : 6},\n                                 mask_type = 'sparsemax',\n                                 seed = CFG.seed)\n\n        ## train\n        model.fit(np.array(X_train),\n                  np.array(y_train.values.ravel()),\n                  eval_set = [(np.array(X_valid), np.array(y_valid.values.ravel()))],\n                  max_epochs = 50,\n                  patience = 10,\n                  batch_size = 2048,\n                  eval_metric = ['auc', 'accuracy', Amex_tabnet])\n\n        models.append(model)\n\n        end = time.time()\n        time_delta = np.round((end - start)/60, 2)\n     \n        print(f'\\nFold {k+1}/{CFG.N_folds} | {time_delta:.2f} min')\n\n        ### free memory\n        del X_train, y_train\n        del X_valid, y_valid\n        gc.collect()\n\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:55.005810Z","iopub.execute_input":"2022-06-17T07:11:55.006271Z","iopub.status.idle":"2022-06-17T07:11:55.231749Z","shell.execute_reply.started":"2022-06-17T07:11:55.006233Z","shell.execute_reply":"2022-06-17T07:11:55.230818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n### RUN TRAINING: \nmodels = run_training()","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:11:55.233289Z","iopub.execute_input":"2022-06-17T07:11:55.233725Z","iopub.status.idle":"2022-06-17T07:20:55.513459Z","shell.execute_reply.started":"2022-06-17T07:11:55.233685Z","shell.execute_reply":"2022-06-17T07:20:55.512640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:20:55.514901Z","iopub.execute_input":"2022-06-17T07:20:55.515472Z","iopub.status.idle":"2022-06-17T07:20:55.530370Z","shell.execute_reply.started":"2022-06-17T07:20:55.515434Z","shell.execute_reply":"2022-06-17T07:20:55.529199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importances","metadata":{}},{"cell_type":"code","source":"features_importances = models[-1].feature_importances_\nargsort = np.argsort(features_importances)\nfeatures_importances_sorted = features_importances[argsort]\n\nfeature_names = train[all_features].columns\nfeatures_sorted = feature_names[argsort]\n\n# plot feature importances\nplt.figure(figsize = (12, 16))\n\n### n features to plot\nn = 50\n\nplt.barh(features_sorted[-n:], features_importances_sorted[-n:])\nplt.title(f\"Feature Importances: {CFG.model}\");","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:20:55.531675Z","iopub.status.idle":"2022-06-17T07:20:55.532344Z","shell.execute_reply.started":"2022-06-17T07:20:55.532110Z","shell.execute_reply":"2022-06-17T07:20:55.532133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"psutil.virtual_memory().percent","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:20:55.533558Z","iopub.status.idle":"2022-06-17T07:20:55.534218Z","shell.execute_reply.started":"2022-06-17T07:20:55.533984Z","shell.execute_reply":"2022-06-17T07:20:55.534007Z"},"trusted":true},"execution_count":null,"outputs":[]}]}