{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Reference\nhttps://www.kaggle.com/code/medali1992/amex-tabnetclassifier-feature-eng-0-791/notebook CV 0.78920 LB 0.791\n\nhttps://www.kaggle.com/code/yekenot/amex-deeptables CV 0.78864 LB 0.791\n\nhttps://www.kaggle.com/code/cascadinglight/tabtransformer-gauss-rank-baseline-kfold\n\nhttps://github.com/lucidrains/tab-transformer-pytorch","metadata":{}},{"cell_type":"code","source":"!pip install tab-transformer-pytorch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom colorama import Fore, Style\nimport random\nimport pickle\nimport time\nimport sys\nimport os\nimport gc\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import QuantileTransformer, KBinsDiscretizer\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchmetrics import AUROC\nfrom tab_transformer_pytorch import TabTransformer\n\nimport warnings\nwarnings.filterwarnings('ignore')\ngc.enable()\n\nprint('PyTorch version',torch.__version__)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    n_folds = 5\n    batch_size = 224\n    epochs = 3\n    lr = 3e-5\n    weight_decay = 0.01\n    n_bins = 10\n    checkpoint = lambda fold: f'full_cat_{fold}.pt'\n    device = 'cuda'\n\n\ntransformer_cfg = {         # cont    # cat  # Number of continuous and categorical features\n    'categories': [CFG.n_bins]*396 + [2]*39, # Iterable with the number of unique values for cat feat\n    'num_continuous': 1,                     # Continuous dimensions in data\n    'dim': 36,                               # Hidden dim, paper set at 32\n    'dim_out': 1,                            # Binary prediction\n    'depth': 4,                              # Depth, paper recommended 6\n    'heads': 6,                              # Heads, paper recommends 8\n    'attn_dropout': 0.05,                    # Post-attention dropout\n    'ff_dropout': 0.15,                      # Feed forward dropout\n    'mlp_hidden_mults': (4, 2),              # Relat mult of each hidden dim of the last mlp to logits\n    'mlp_act': nn.GELU(),                    # Activation for final mlp, defaults to relu\n    'continuous_mean_std': torch.randn(1, 2) # Norm the continuous values before layer norm (optional)\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True       \nseed_everything(seed=CFG.seed)\n\n\ndef amex_metric_mod(y_true, y_pred):\n\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n        \n    return 0.5 * (gini[1]/gini[0] + top_four)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_11', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_2', \n                'B_20', 'B_28', 'B_29', 'B_3', 'B_33', 'B_36', 'B_37', 'B_4', 'B_42', \n                'B_5', 'B_8', 'B_9', 'D_102', 'D_103', 'D_105', 'D_111', 'D_112', 'D_113', \n                'D_115', 'D_118', 'D_119', 'D_121', 'D_124', 'D_128', 'D_129', 'D_131', \n                'D_132', 'D_133', 'D_139', 'D_140', 'D_141', 'D_143', 'D_144', 'D_145', \n                'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', \n                'D_49', 'D_50', 'D_51', 'D_52', 'D_56', 'D_58', 'D_62', 'D_70', 'D_71', \n                'D_72', 'D_74', 'D_75', 'D_79', 'D_81', 'D_83', 'D_84', 'D_88', 'D_91', \n                'P_2', 'P_3', 'R_1', 'R_10', 'R_11', 'R_13', 'R_18', 'R_19', 'R_2', 'R_26', \n                'R_27', 'R_28', 'R_3', 'S_11', 'S_12', 'S_22', 'S_23', 'S_24', 'S_26', \n                'S_27', 'S_5', 'S_7', 'S_8', ]\nfeatures_min = ['B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_2', 'B_20', 'B_22', \n                'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_33', 'B_36', 'B_4', 'B_42', \n                'B_5', 'B_9', 'D_102', 'D_103', 'D_107', 'D_109', 'D_110', 'D_111', \n                'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', \n                'D_129', 'D_132', 'D_133', 'D_139', 'D_140', 'D_141', 'D_143', 'D_144', \n                'D_145', 'D_39', 'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', \n                'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', \n                'D_71', 'D_74', 'D_75', 'D_78', 'D_79', 'D_81', 'D_83', 'D_84', 'D_86', \n                'D_88', 'D_96', 'P_2', 'P_3', 'P_4', 'R_1', 'R_11', 'R_13', 'R_17', 'R_19', \n                'R_2', 'R_27', 'R_28', 'R_4', 'R_5', 'R_8', 'S_11', 'S_12', 'S_23', 'S_25', \n                'S_3', 'S_5', 'S_7', 'S_9', ]\nfeatures_max = ['B_1', 'B_11', 'B_13', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_2', \n                'B_22', 'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_31', 'B_33', 'B_36', \n                'B_4', 'B_42', 'B_5', 'B_7', 'B_9', 'D_102', 'D_103', 'D_105', 'D_109', \n                'D_110', 'D_112', 'D_113', 'D_115', 'D_121', 'D_124', 'D_128', 'D_129', \n                'D_131', 'D_139', 'D_141', 'D_144', 'D_145', 'D_39', 'D_41', 'D_42', \n                'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_52', \n                'D_53', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_72', 'D_74', \n                'D_75', 'D_79', 'D_81', 'D_83', 'D_84', 'D_88', 'D_89', 'P_2', 'P_3', \n                'R_1', 'R_10', 'R_11', 'R_26', 'R_28', 'R_3', 'R_4', 'R_5', 'R_7', 'R_8', \n                'S_11', 'S_12', 'S_23', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_7', 'S_8', ]\nfeatures_last = ['B_1', 'B_11', 'B_12', 'B_13', 'B_14', 'B_16', 'B_18', 'B_19', 'B_2', \n                 'B_20', 'B_21', 'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_30', 'B_31', \n                 'B_33', 'B_36', 'B_37', 'B_38', 'B_39', 'B_4', 'B_40', 'B_42', 'B_5', \n                 'B_8', 'B_9', 'D_102', 'D_105', 'D_106', 'D_107', 'D_108', 'D_110', \n                 'D_111', 'D_112', 'D_113', 'D_114', 'D_115', 'D_116', 'D_117', 'D_118', \n                 'D_119', 'D_120', 'D_121', 'D_124', 'D_126', 'D_128', 'D_129', 'D_131', \n                 'D_132', 'D_133', 'D_137', 'D_138', 'D_139', 'D_140', 'D_141', 'D_142', \n                 'D_143', 'D_144', 'D_145', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', \n                 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_55', \n                 'D_56', 'D_59', 'D_60', 'D_62', 'D_63', 'D_64', 'D_66', 'D_68', 'D_70', \n                 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_77', 'D_78', 'D_81', 'D_82', \n                 'D_83', 'D_84', 'D_88', 'D_89', 'D_91', 'D_94', 'D_96', 'P_2', 'P_3', \n                 'P_4', 'R_1', 'R_10', 'R_11', 'R_12', 'R_13', 'R_16', 'R_17', 'R_18', \n                 'R_19', 'R_25', 'R_28', 'R_3', 'R_4', 'R_5', 'R_8', 'S_11', 'S_12', \n                 'S_23', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_9', ]\nfeatures_categorical = ['B_30_last', 'B_38_last', 'D_114_last', 'D_116_last',\n                        'D_117_last', 'D_120_last', 'D_126_last',\n                        'D_63_last', 'D_64_last', 'D_66_last', 'D_68_last']\n\nINFERENCE = True\nfor i in [0, 1] if INFERENCE else [0]:\n    # i == 0 -> process the train data\n    # i == 1 -> process the test data\n    df = pd.read_feather(['../input/amexfeather/train_data.ftr',\n                          '../input/amexfeather/test_data.ftr'][i])\n    cid = pd.Categorical(df.pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # Mask for last statement of every customer\n    if i == 0: # train\n        target = df.loc[last, 'target']\n    print('Read', i)\n    gc.collect()\n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    print('Computed avg', i)\n    gc.collect()\n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    print('Computed max', i)\n    gc.collect()\n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    print('Computed min', i)\n    gc.collect()\n    df_last = (df.loc[last, features_last]\n               .rename(columns={f: f\"{f}_last\" for f in features_last})\n               .set_index(np.asarray(cid[last]))\n              )\n    df = None # We no longer need the original data\n    print('Computed last', i)\n    \n    df_categorical = df_last[features_categorical].astype(object)\n    features_not_cat = [f for f in df_last.columns if f not in features_categorical]\n    if i == 0: # train\n        ohe = OneHotEncoder(drop='first', sparse=False, dtype=np.float32, handle_unknown='ignore')\n        ohe.fit(df_categorical)\n        with open(\"ohe.pickle\", 'wb') as f: pickle.dump(ohe, f)\n    df_categorical = pd.DataFrame(ohe.transform(df_categorical).astype(np.float16),\n                                  index=df_categorical.index).rename(columns=str)\n    print('Computed categorical', i)\n    \n    categorical_columns = df_categorical.columns.values.tolist()\n    print('Categorical columns num', len(categorical_columns))\n    \n    df = pd.concat([df_last[features_not_cat], df_avg, df_min, df_max, df_categorical], axis=1)\n    \n    # Impute missing values\n    df.fillna(value=0, inplace=True)\n    \n    del df_avg, df_max, df_min, df_last, df_categorical, cid, last, features_not_cat\n    \n    if i == 0: # train\n        # Free the memory\n        df.reset_index(drop=True, inplace=True) # Frees 0.2 GByte\n        df.to_feather('train_processed.ftr')\n        df = None\n        gc.collect()\n        \ntrain = pd.read_feather('train_processed.ftr')\ntest = df\ntarget = target.reset_index(drop=True)\ndel df, ohe\ngc.collect()\n\nprint('Train Shapes:', train.shape, target.shape)\nif INFERENCE: print('Test Shapes:', test.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nall_columns = train.columns.values.tolist()\ncont_cols = list(set(all_columns).symmetric_difference(set(categorical_columns)))\ncat_cols = categorical_columns\nprint('cont cols number', len(cont_cols))\nprint('cat cols number', len(cat_cols))\ntrain['target'] = target\n\nprint('Scaling')\ntrain_scaled, test_scaled = train.copy(), test.copy()\nfor col in cont_cols:\n    transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution=\"normal\")\n    vec_len = len(train_scaled[col].values)\n    vec_len_test = len(test_scaled[col].values)\n    raw_vec = train_scaled[col].values.reshape(vec_len, 1)\n    transformer.fit(raw_vec)\n\n    train_scaled[col] = transformer.transform(raw_vec).reshape(1, vec_len)[0]\n    test_scaled[col] = transformer.transform(test_scaled[col].values.reshape(vec_len_test,\n                                                                 1)).reshape(1, vec_len_test)[0]\n    \n\nfig, axs = plt.subplots(2)\naxs[0].title.set_text('Feat \"B_1_last\" dist before Gauss Rank')\naxs[0].hist(train['B_1_last'], bins=100)\n\naxs[1].title.set_text('Feat \"B_1_last\" dist after Gauss Rank')\naxs[1].hist(train_scaled['B_1_last'], bins=100)\nplt.tight_layout()\nplt.show()\n\ndel train, test\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndisc = KBinsDiscretizer(n_bins=CFG.n_bins, encode='ordinal', strategy='uniform')\nprint('Discretizing')\ntrain_scaled[cont_cols] = disc.fit_transform(train_scaled[cont_cols])\ntest_scaled[cont_cols] = disc.transform(test_scaled[cont_cols])\n\ntarget = train_scaled.target.values\ntrain_scaled = train_scaled.drop('target', 1).values\ntest_scaled = test_scaled.values","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter:\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \n        \nclass EarlyStopping:\n    def __init__(self, patience=3, mode=\"max\", delta=0.001, verbose=None):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.delta = delta\n        self.verbose = verbose\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n    def __call__(self, epoch_score, model, model_path):\n\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score:  # + self.delta\n            self.counter += 1\n            if self.verbose:\n                print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model, model_path)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            if self.verbose:\n                print('Validation score improved ({:.4f} --> {:.4f}). Saving model!'.format(self.val_score, epoch_score))\n                \n            torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score\n        \n        \nclass TabDataset(Dataset):\n    def __init__(self, cat, target=None):\n        super().__init__()\n        self.cat = cat\n        self.target = target\n        \n    def __len__(self):\n        return len(self.cat)\n    \n    def __getitem__(self, idx):\n        cat = self.cat[idx]\n        \n        _dict = {'cont': torch.ones(1),\n                 'cat': torch.LongTensor(cat)}\n        \n        if self.target is not None:\n            target = self.target[idx].item()\n            _dict.update({'target': torch.tensor(target, dtype = torch.float)})\n        \n        return _dict\n    \n    \nclass Trainer:\n    def __init__(self, model, device, loss_fn, opt, scheduler=None):\n        self.model = model\n        self.device = device\n        self.loss_fn = loss_fn\n        self.opt = opt\n        self.scheduler = scheduler\n        \n    def fit_one_epoch(self, dl):\n        self.model.train()\n        losses = AverageMeter()\n        prog_bar = tqdm(enumerate(dl), total = len(dl), file=sys.stdout, leave=False)\n        \n        for bi, d in prog_bar:\n            cont = d[\"cont\"].to(self.device)\n            cat = d['cat'].to(self.device)\n            target = d['target'].to(self.device)\n            \n            out = self.model(cat, cont)\n            loss = self.loss_fn(out.squeeze(-1), target)\n            prog_bar.set_description('loss: {:.2f}'.format(loss.item()))\n            losses.update(loss.item(), cont.size(0))\n            loss.backward()\n            self.opt.step()\n            \n            if self.scheduler: \n                self.scheduler.step()\n                    \n            self.opt.zero_grad()\n            \n    def eval_one_epoch(self, dl, **kwargs):\n        self.model.eval()\n        losses = AverageMeter()\n        amex_scores = AverageMeter()\n        metric = AUROC()\n        prog_bar = tqdm(enumerate(dl), total=len(dl), file=sys.stdout, leave=False)\n        oof_df = pd.DataFrame()\n        \n        for bi, d in prog_bar:  \n            cont = d[\"cont\"].to(self.device)\n            cat = d['cat'].to(self.device)\n            target = d['target'].to(self.device)\n            \n            with torch.no_grad():\n                out = self.model(cat, cont)\n                loss = self.loss_fn(out.squeeze(-1), target)\n                auroc = metric(out.squeeze(-1), target.int())\n                amex_metric = amex_metric_mod(target.int().cpu().detach().numpy(), out.squeeze(-1).cpu().detach().numpy())\n                \n                losses.update(loss.item(), cont.size(0))\n                amex_scores.update(amex_metric.item(), cont.size(0))\n                oof_df = pd.concat([oof_df, pd.DataFrame(out.squeeze(-1).cpu().detach().numpy())])\n                \n        auroc = metric.compute()\n        print(f\"Epoch {str(kwargs['epoch']):2s}\"\\\n              f\"  Valid Loss: {losses.avg:.4f}  AUROC Score: {auroc:.4f}  Amex Score: {amex_scores.avg:.4f}\")\n        return auroc.cpu() if metric else losses.avg, oof_df.values.flatten()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed)\nsplits = [*kfold.split(train_scaled, target)]\noof_predictions = np.zeros((train_scaled.shape[0]))\n\ndef create_dataloaders(fold):\n    train_idx, valid_idx = splits[fold]\n    \n    _xtr, _ytr = train_scaled[train_idx], target[train_idx]\n    _xval, _yval = train_scaled[valid_idx], target[valid_idx]\n    \n    train_ds = TabDataset(cat = _xtr, target = _ytr)\n    valid_ds = TabDataset(cat = _xval, target = _yval)\n                          \n    train_dl = DataLoader(train_ds, batch_size = CFG.batch_size, shuffle = True)\n    valid_dl = DataLoader(valid_ds, batch_size = CFG.batch_size, shuffle = False)\n    \n    return train_dl, valid_dl, valid_idx\n\n\ndef train_fold(fold, epochs = 10):\n    print(f\"**************************** \")\n    print(f\"********** Fold {fold+1} ********** \",'\\n')\n    train_dl, valid_dl, valid_idx = create_dataloaders(fold)\n    es = EarlyStopping(patience = 1, mode=\"max\", verbose = False)\n    \n    model = TabTransformer(**transformer_cfg).to(CFG.device)\n       \n    opt = torch.optim.AdamW(model.parameters(), lr = 5e-5, weight_decay = CFG.weight_decay)\n    scheduler = CosineAnnealingLR(opt, T_max=5, eta_min=1e-5)\n    \n    trainer = Trainer(model, \n                      CFG.device, \n                      loss_fn=nn.BCEWithLogitsLoss(),\n                      opt = opt,\n                      scheduler = scheduler,\n                     )\n    \n    for epoch in range(epochs):\n        trainer.fit_one_epoch(train_dl)\n        valid_loss, preds = trainer.eval_one_epoch(valid_dl, fold = fold, epoch = epoch)\n        \n        es(valid_loss, trainer.model, model_path = CFG.checkpoint(fold))\n        oof_predictions[valid_idx] = oof_predictions[valid_idx] + preds\n        \n        if es.early_stop:\n            break\n            \n    del train_dl, valid_dl, model, trainer\n    gc.collect()\n    \n    \nfor fold in range(CFG.n_folds):\n    start = time.time()\n    \n    train_fold(fold, CFG.epochs)\n    \n    end = time.time()\n    time_delta = np.round((end - start)/60, 2)\n    print(f'{Fore.GREEN}{Style.BRIGHT}\\nFold {fold+1}/{CFG.n_folds} | {time_delta:.2f} min{Style.RESET_ALL}','\\n')    \n    torch.cuda.empty_cache()\n    _ = gc.collect()\n\nprint(f'OOF Amex metric across folds: {amex_metric_mod(target.flatten(), oof_predictions.flatten())}')\ndel train_scaled\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = torch.zeros(len(test_scaled), 1).to(CFG.device)\ntest_ds = TabDataset(cat = test_scaled)\ntest_dl = DataLoader(test_ds, batch_size = CFG.batch_size, shuffle = False)\n\nwith torch.no_grad():\n    for fold in range(CFG.n_folds):\n        preds = []\n        model = TabTransformer(**transformer_cfg).to(CFG.device)\n        state_dict = CFG.checkpoint(fold)\n        model.load_state_dict(torch.load(state_dict))\n        model.eval()\n        \n        for d in test_dl:\n            cont = d[\"cont\"].to(CFG.device)\n            cat = d['cat'].to(CFG.device)\n            out = model(cat, cont)\n            preds.append(out)\n            \n        preds = torch.vstack(preds)\n        y_pred += preds / CFG.n_folds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsub.iloc[:,1] = y_pred.cpu()\nsub = sub.set_index('customer_ID')\nsub.to_csv('submission.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}