{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PyTorch Quickstart for the AMEX Competition: Training and Inference\n\nThe notebook is based on implementation of the [Keras Quickstart for the AMEX Competition](https://www.kaggle.com/code/ambrosm/amex-keras-quickstart-1-training)\n\nThis notebook shows\n- how to do space-efficient feature engineering\n- how to implement a simple PyTorch model\n- how to train and cross-validate the model","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport pickle\nfrom matplotlib import pyplot as plt\nimport random\nimport datetime\nimport math\nfrom matplotlib.ticker import MaxNLocator\nfrom colorama import Fore, Back, Style\nimport gc\nimport copy\n\nfrom sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer, OneHotEncoder\nfrom sklearn.metrics import roc_curve, roc_auc_score, average_precision_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import MultiStepLR\n\nINFERENCE = True","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2022-06-02T14:19:31.264072Z","iopub.status.busy":"2022-06-02T14:19:31.263587Z","iopub.status.idle":"2022-06-02T14:19:39.017427Z","shell.execute_reply":"2022-06-02T14:19:39.01651Z","shell.execute_reply.started":"2022-06-02T14:19:31.263977Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n    print(\"G: {:.6f}, D: {:.6f}, ALL: {:6f}\".format(g, d, 0.5*(g+d)))\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2022-05-29T17:11:14.752788Z","iopub.status.busy":"2022-05-29T17:11:14.752372Z","iopub.status.idle":"2022-05-29T17:11:14.774426Z","shell.execute_reply":"2022-05-29T17:11:14.77346Z","shell.execute_reply.started":"2022-05-29T17:11:14.752751Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the training data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we create several groups of features:\n- Selected features averaged over all statements of a customer\n- Minimum / maximum of selected features over all statements of a customer\n- Selected features taken from the last statement of a customer\n\nWe one-hot encode the categorical features and fill all missing values with 0.\n\nThe code has been optimized for memory efficiency rather than readability. In particular, `.iloc[mask_array, columns]` needs much less RAM than the groupby construction used in previous versions of the notebook. Deleting the index of the train dataframe frees another 0.2 GByte.\n","metadata":{}},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_11', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_2', \n                'B_20', 'B_28', 'B_29', 'B_3', 'B_33', 'B_36', 'B_37', 'B_4', 'B_42', \n                'B_5', 'B_8', 'B_9', 'D_102', 'D_103', 'D_105', 'D_111', 'D_112', 'D_113', \n                'D_115', 'D_118', 'D_119', 'D_121', 'D_124', 'D_128', 'D_129', 'D_131', \n                'D_132', 'D_133', 'D_139', 'D_140', 'D_141', 'D_143', 'D_144', 'D_145', \n                'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', \n                'D_49', 'D_50', 'D_51', 'D_52', 'D_56', 'D_58', 'D_62', 'D_70', 'D_71', \n                'D_72', 'D_74', 'D_75', 'D_79', 'D_81', 'D_83', 'D_84', 'D_88', 'D_91', \n                'P_2', 'P_3', 'R_1', 'R_10', 'R_11', 'R_13', 'R_18', 'R_19', 'R_2', 'R_26', \n                'R_27', 'R_28', 'R_3', 'S_11', 'S_12', 'S_22', 'S_23', 'S_24', 'S_26', \n                'S_27', 'S_5', 'S_7', 'S_8', ]\nfeatures_min = ['B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_2', 'B_20', 'B_22', \n                'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_33', 'B_36', 'B_4', 'B_42', \n                'B_5', 'B_9', 'D_102', 'D_103', 'D_107', 'D_109', 'D_110', 'D_111', \n                'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', \n                'D_129', 'D_132', 'D_133', 'D_139', 'D_140', 'D_141', 'D_143', 'D_144', \n                'D_145', 'D_39', 'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', \n                'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', \n                'D_71', 'D_74', 'D_75', 'D_78', 'D_79', 'D_81', 'D_83', 'D_84', 'D_86', \n                'D_88', 'D_96', 'P_2', 'P_3', 'P_4', 'R_1', 'R_11', 'R_13', 'R_17', 'R_19', \n                'R_2', 'R_27', 'R_28', 'R_4', 'R_5', 'R_8', 'S_11', 'S_12', 'S_23', 'S_25', \n                'S_3', 'S_5', 'S_7', 'S_9', ]\nfeatures_max = ['B_1', 'B_11', 'B_13', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_2', \n                'B_22', 'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_31', 'B_33', 'B_36', \n                'B_4', 'B_42', 'B_5', 'B_7', 'B_9', 'D_102', 'D_103', 'D_105', 'D_109', \n                'D_110', 'D_112', 'D_113', 'D_115', 'D_121', 'D_124', 'D_128', 'D_129', \n                'D_131', 'D_139', 'D_141', 'D_144', 'D_145', 'D_39', 'D_41', 'D_42', \n                'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_52', \n                'D_53', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_72', 'D_74', \n                'D_75', 'D_79', 'D_81', 'D_83', 'D_84', 'D_88', 'D_89', 'P_2', 'P_3', \n                'R_1', 'R_10', 'R_11', 'R_26', 'R_28', 'R_3', 'R_4', 'R_5', 'R_7', 'R_8', \n                'S_11', 'S_12', 'S_23', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_7', 'S_8', ]\nfeatures_last = ['B_1', 'B_11', 'B_12', 'B_13', 'B_14', 'B_16', 'B_18', 'B_19', 'B_2', \n                 'B_20', 'B_21', 'B_24', 'B_27', 'B_28', 'B_29', 'B_3', 'B_30', 'B_31', \n                 'B_33', 'B_36', 'B_37', 'B_38', 'B_39', 'B_4', 'B_40', 'B_42', 'B_5', \n                 'B_8', 'B_9', 'D_102', 'D_105', 'D_106', 'D_107', 'D_108', 'D_110', \n                 'D_111', 'D_112', 'D_113', 'D_114', 'D_115', 'D_116', 'D_117', 'D_118', \n                 'D_119', 'D_120', 'D_121', 'D_124', 'D_126', 'D_128', 'D_129', 'D_131', \n                 'D_132', 'D_133', 'D_137', 'D_138', 'D_139', 'D_140', 'D_141', 'D_142', \n                 'D_143', 'D_144', 'D_145', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', \n                 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_55', \n                 'D_56', 'D_59', 'D_60', 'D_62', 'D_63', 'D_64', 'D_66', 'D_68', 'D_70', \n                 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_77', 'D_78', 'D_81', 'D_82', \n                 'D_83', 'D_84', 'D_88', 'D_89', 'D_91', 'D_94', 'D_96', 'P_2', 'P_3', \n                 'P_4', 'R_1', 'R_10', 'R_11', 'R_12', 'R_13', 'R_16', 'R_17', 'R_18', \n                 'R_19', 'R_25', 'R_28', 'R_3', 'R_4', 'R_5', 'R_8', 'S_11', 'S_12', \n                 'S_23', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_9', ]\nfeatures_categorical = ['B_30_last', 'B_38_last', 'D_114_last', 'D_116_last',\n                        'D_117_last', 'D_120_last', 'D_126_last',\n                        'D_63_last', 'D_64_last', 'D_66_last', 'D_68_last']\n\nfor i in [0, 1] if INFERENCE else [0]:\n    # i == 0 -> process the train data\n    # i == 1 -> process the test data\n    df = pd.read_feather(['../input/amexfeather/train_data.ftr',\n                          '../input/amexfeather/test_data.ftr'][i])\n    cid = pd.Categorical(df.pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if i == 0: # train\n        target = df.loc[last, 'target']\n    print('Read', i)\n    gc.collect()\n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    print('Computed avg', i)\n    gc.collect()\n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    print('Computed max', i)\n    gc.collect()\n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    print('Computed min', i)\n    gc.collect()\n    df_last = (df.loc[last, features_last]\n               .rename(columns={f: f\"{f}_last\" for f in features_last})\n               .set_index(np.asarray(cid[last]))\n              )\n    df = None # we no longer need the original data\n    print('Computed last', i)\n    \n    df_categorical = df_last[features_categorical].astype(object)\n    features_not_cat = [f for f in df_last.columns if f not in features_categorical]\n    if i == 0: # train\n        ohe = OneHotEncoder(drop='first', sparse=False, dtype=np.float32, handle_unknown='ignore')\n        ohe.fit(df_categorical)\n        with open(\"ohe.pickle\", 'wb') as f: pickle.dump(ohe, f)\n    df_categorical = pd.DataFrame(ohe.transform(df_categorical).astype(np.float16),\n                                  index=df_categorical.index).rename(columns=str)\n    print('Computed categorical', i)\n    \n    df = pd.concat([df_last[features_not_cat], df_categorical, df_avg, df_min, df_max], axis=1)\n    \n    # Impute missing values\n    df.fillna(value=0, inplace=True)\n    \n    del df_avg, df_max, df_min, df_last, df_categorical, cid, last, features_not_cat\n    \n    if i == 0: # train\n        # Free the memory\n        df.reset_index(drop=True, inplace=True) # frees 0.2 GByte\n        df.to_feather('train_processed.ftr')\n        df = None\n        gc.collect()\n\ntrain = pd.read_feather('train_processed.ftr')\ntest = df\ndel df, ohe\n\nprint('Shapes:', train.shape, target.shape)","metadata":{"execution":{"iopub.execute_input":"2022-05-29T17:11:14.777796Z","iopub.status.busy":"2022-05-29T17:11:14.776866Z","iopub.status.idle":"2022-05-29T17:16:01.186479Z","shell.execute_reply":"2022-05-29T17:16:01.185039Z","shell.execute_reply.started":"2022-05-29T17:11:14.777753Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The model\n\nOur model has four hidden layers, enriched by a skip connection and a Dropout layer.","metadata":{}},{"cell_type":"code","source":"class my_model(nn.Module):\n    def __init__(self, in_feat, hid_dim=256, activation=nn.ReLU(), dropout=0.1):\n        super(my_model, self).__init__()\n        self.encode = nn.Linear(in_feat, hid_dim)\n        self.activation = activation\n        self.hidden1 = nn.Linear(hid_dim, 64)\n        self.hidden2 = nn.Linear(64, 64)\n        self.drop = nn.Dropout(dropout)\n        self.hidden3 = nn.Linear(64+hid_dim, 16)\n        self.pred = nn.Linear(16, 2)\n    \n    def forward(self, x):\n        h0 = self.activation(self.encode(x))\n        h1 = self.activation(self.hidden2(self.activation(self.hidden1(h0))))\n        h = self.drop(torch.concat([h0, h1], dim=-1))\n        h = self.activation(self.hidden3(h))\n        return self.pred(h)","metadata":{"execution":{"iopub.execute_input":"2022-05-29T17:16:01.18838Z","iopub.status.busy":"2022-05-29T17:16:01.188016Z","iopub.status.idle":"2022-05-29T17:16:02.888402Z","shell.execute_reply":"2022-05-29T17:16:02.88705Z","shell.execute_reply.started":"2022-05-29T17:16:01.18835Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Early Stopper","metadata":{}},{"cell_type":"code","source":"class early_stopper(object):\n    def __init__(self, patience=12, verbose=False, delta=0):\n        self.patience = patience\n        self.verbose = verbose\n        self.delta = delta\n        self.best_value = None\n        self.best_cv = None\n        self.is_earlystop = False\n        self.count = 0\n        self.best_model = None\n        #self.val_preds = []\n        #self.val_logits = []\n\n    def earlystop(self, loss, value, model=None):#, preds, logits):\n        \"\"\"\n        value: evaluation value on valiation dataset\n        \"\"\"\n        cv = value\n        if self.best_value is None:\n            self.best_value = value\n            self.best_cv = cv\n            self.best_model = copy.deepcopy(model).to('cpu')\n            #self.val_preds = preds\n            #self.val_logits = logits\n        elif value < self.best_value + self.delta:\n            self.count += 1\n            if self.verbose:\n                print('EarlyStoper count: {:02d}'.format(self.count))\n            if self.count >= self.patience:\n                self.is_earlystop = True\n        else:\n            self.best_value = value\n            self.best_cv = cv\n            self.best_model = copy.deepcopy(model).to('cpu')\n            #self.val_preds = preds\n            #self.val_logits = logits\n            self.count = 0\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross-validation\n\nWe use a standard cross-validation loop. In the loop, we scale the data and train a model. We use a StratifiedKFold because the data is imbalanced.\n","metadata":{}},{"cell_type":"code","source":"def fit_model(train_nn, train_y, test_nn, params):\n    device = params['device']\n    oof_predictions = torch.from_numpy(np.zeros([len(train_nn), 2])).float().to(device)\n    test_predictions = torch.from_numpy(np.zeros([len(test_nn), 2])).float().to(device)\n    kfold = StratifiedKFold(n_splits=params['n_fold'], shuffle=True, random_state=params['seed'])\n    features_numerical = [f for f in train_nn.columns if f != 'target' and f != 'customer_ID']\n    y_target = train_y.target.values\n    num_feat = train_nn[features_numerical]\n    y = train_y.target\n    labels = torch.from_numpy(y.values).long().to(device)\n    loss_fn = nn.CrossEntropyLoss(weight=torch.from_numpy(np.array([118828, 340085])).float()).to(device)\n    for fold, (trn_idx, val_idx) in enumerate(kfold.split(train_nn, y_target)):\n        print(f'Training fold {fold + 1}')\n        x_train, x_val = num_feat.iloc[trn_idx], num_feat.iloc[val_idx]\n        scaler = StandardScaler()\n        x_train = torch.from_numpy(scaler.fit_transform(x_train)).float().to(device)\n        x_val = torch.from_numpy(scaler.transform(x_val)).float().to(device)\n        y_train, y_val = labels[trn_idx], labels[val_idx]\n        train_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(trn_idx)),\n                                                                               num_samples=len(trn_idx), replacement=False)\n        train_dataloader = torch.utils.data.DataLoader(np.array(range(len(trn_idx))), batch_size=params['batch_size'], num_workers=0,\n                                                       sampler=train_sample_strategy, drop_last=False)\n        val_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(val_idx)),\n                                                                             num_samples=len(val_idx), replacement=False)\n        val_dataloader = torch.utils.data.DataLoader(np.array(range(len(val_idx))), batch_size=params['batch_size'], num_workers=0,\n                                                     sampler=val_sample_strategy, drop_last=False)\n        model = eval(params['model'])(x_train.shape[1]).to(device)\n        lr = params['lr'] * np.sqrt(params['batch_size']/2048)\n        optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=params['wd'])\n        lr_scheduler = MultiStepLR(optimizer=optimizer, milestones=[800, 1600, 2400, 3200, 4000, 4800, 5600, 6400, 7200], gamma=0.6)\n        earlystoper = early_stopper(patience=params['early_stopping'], verbose=True)\n        start_epoch = 0\n        for epoch in range(start_epoch, params['max_epochs']):\n            train_loss_list = []\n            # train_acc_list = []\n            model.train()\n            for step, input_seeds in enumerate(train_dataloader):\n                batch_inputs = x_train[input_seeds].to(device)\n                batch_labels = y_train[input_seeds].to(device)\n                train_batch_logits = model(batch_inputs)\n                train_loss = loss_fn(train_batch_logits, batch_labels)\n                # backward\n                optimizer.zero_grad()\n                train_loss.backward()\n                optimizer.step()\n                lr_scheduler.step()\n                train_loss_list.append(train_loss.cpu().detach().numpy())\n                \n                #tr_batch_pred = None\n\n                if step % 10 == 0:\n                    tr_batch_pred = torch.sum(torch.argmax(train_batch_logits.clone().detach(), dim=1) == batch_labels) / batch_labels.shape[0]\n                    score = torch.softmax(train_batch_logits.clone().detach(), dim=1)[:, 1].cpu().numpy()\n                    print('In epoch:{:03d}|batch:{:04d}, train_loss:{:4f}, '\n                          'train_ap:{:.4f}, train_acc:{:.4f}, train_auc:{:.4f}'.format(epoch,step,\n                                                                                       np.mean(train_loss_list),\n                                                                                       average_precision_score(batch_labels.cpu().numpy(), score), \n                                                                                       tr_batch_pred.detach(),\n                                                                                       roc_auc_score(batch_labels.cpu().numpy(), score)))\n        \n            # mini-batch for validation\n            val_loss_list = 0\n            val_acc_list = 0\n            #val_correct_list = 0\n            val_all_list = 0\n            model.eval()\n            with torch.no_grad():\n                for step, input_seeds in enumerate(val_dataloader):\n                    batch_inputs = x_val[input_seeds].to(device)\n                    batch_labels = y_val[input_seeds].to(device)\n                    val_batch_logits = model(batch_inputs)\n                    oof_predictions[val_idx[input_seeds]] = val_batch_logits\n                    val_loss_list = val_loss_list + loss_fn(val_batch_logits, batch_labels)\n                    val_batch_pred = torch.sum(torch.argmax(val_batch_logits, dim=1) == batch_labels) / torch.tensor(batch_labels.shape[0])\n                    val_acc_list = val_acc_list + val_batch_pred * torch.tensor(batch_labels.shape[0])\n                    val_all_list = val_all_list + batch_labels.shape[0]\n                    if step % 10 == 0:\n                        score = torch.softmax(val_batch_logits.clone().detach(), dim=1)[:, 1].cpu().numpy()\n                        print('In epoch:{:03d}|batch:{:04d}, val_loss:{:4f}, val_ap:{:.4f}, '\n                              'val_acc:{:.4f}, val_auc:{:.4f}'.format(epoch,\n                                                                      step,\n                                                                      val_loss_list/val_all_list,\n                                                                      average_precision_score(batch_labels.cpu().numpy(), score), \n                                                                      val_batch_pred.detach(),\n                                                                      roc_auc_score(batch_labels.cpu().numpy(), score)))\n                #tmp_predictions = model(test_feature).cpu().numpy()\n            #infold_preds[fold] = tmp_predictions\n            #test_predictions += tmp_predictions / params['n_fold']\n            val_predictions = torch.softmax(oof_predictions[val_idx, :].detach(), dim=-1)[:, 1].cpu().numpy()\n            earlystoper.earlystop(val_loss_list, amex_metric(y_val.float().cpu().numpy(), val_predictions), model)\n            if earlystoper.is_earlystop:\n                print(\"Early Stopping!\")\n                break\n        print(\"Best val_metric is: {:.7f}\".format(earlystoper.best_cv))\n        test_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(test_nn)),\n                                                                              num_samples=len(test_nn), replacement=False)\n        test_dataloader = torch.utils.data.DataLoader(np.array(range(len(test_nn))), batch_size=params['batch_size'], num_workers=0,\n                                                      sampler=test_sample_strategy, drop_last=False)\n        test_num_feat = torch.from_numpy(scaler.transform(test_nn[features_numerical])).float().to(device)\n        b_model = earlystoper.best_model.to(device)\n        b_model.eval()\n        with torch.no_grad():\n            for step, input_seeds in enumerate(test_dataloader):\n                batch_inputs = test_num_feat[input_seeds].to(device)\n                test_batch_logits = b_model(batch_inputs)\n                test_predictions[input_seeds] = test_batch_logits\n                #test_batch_pred = torch.sum(torch.argmax(test_batch_logits, dim=1) == batch_labels) / torch.tensor(batch_labels.shape[0])\n                if step % 10 == 0:\n                    print('In test batch:{:04d}'.format(step))\n    #my_acc = acc(y, oof_predictions)\n    my_ap = average_precision_score(y_target, torch.softmax(oof_predictions, dim=1).cpu()[:, 1])\n    print(\"NN out of fold AP is:\", my_ap)\n    return earlystoper.best_model.to('cpu'), oof_predictions, test_predictions","metadata":{"execution":{"iopub.execute_input":"2022-05-29T17:16:02.891278Z","iopub.status.busy":"2022-05-29T17:16:02.89085Z","iopub.status.idle":"2022-05-29T17:19:16.963266Z","shell.execute_reply":"2022-05-29T17:19:16.961844Z","shell.execute_reply.started":"2022-05-29T17:16:02.891241Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'model': 'my_model',\n    'batch_size': 2048,\n    'lr': 0.01,\n    'wd': 4e-4,\n    #'device': 'cpu',\n    'device': 'cuda:0',\n    'early_stopping': 12,\n    'n_fold': 5,\n    'seed': 2021,\n    'max_epochs': 200,\n}\n\nb_models, val_nn_0, test_nn_0 = fit_model(train_nn=train,\n                                          train_y=pd.DataFrame(target), \n                                          test_nn=test,\n                                          params=params)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nWe submit the mean of the five predictions.","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame({'customer_ID': test.index,\n                    'prediction': torch.softmax(test_nn_0, dim=-1)[:, 1].cpu().numpy()})\nsub.to_csv('submission_mlp_baseline_pytorch.csv', index=False)\ndisplay(sub)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T17:19:39.240302Z","iopub.status.idle":"2022-05-29T17:19:39.240799Z","shell.execute_reply":"2022-05-29T17:19:39.240635Z","shell.execute_reply.started":"2022-05-29T17:19:39.240614Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}