{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PyTorch Logistic Regression x 300\nThis notebook merge the predictions of 300 logistic regression models (5 folds x 3 learning rates x 5 weight decays x 4 repeated models = 300 models)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport copy\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import MultiStepLR","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n    print(\"G: {:.6f}, D: {:.6f}, ALL: {:6f}\".format(g, d, 0.5*(g+d)))\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T03:37:06.40583Z","iopub.execute_input":"2022-06-07T03:37:06.40648Z","iopub.status.idle":"2022-06-07T03:37:06.453573Z","shell.execute_reply.started":"2022-06-07T03:37:06.406409Z","shell.execute_reply":"2022-06-07T03:37:06.452217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LogisticRegression(nn.Module):\n    def __init__(self, in_feats, repeat=1):\n        super(LogisticRegression, self).__init__()\n        self.encode = nn.Linear(in_feats, repeat, bias=True)\n        self.output = nn.Sigmoid()\n    \n    def forward(self, x):\n        return self.output(self.encode(x))","metadata":{"execution":{"iopub.status.busy":"2022-06-07T03:37:06.460575Z","iopub.execute_input":"2022-06-07T03:37:06.464296Z","iopub.status.idle":"2022-06-07T03:37:06.473842Z","shell.execute_reply.started":"2022-06-07T03:37:06.464247Z","shell.execute_reply":"2022-06-07T03:37:06.472923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class early_stopper(object):\n    def __init__(self, patience=12, verbose=False, delta=0):\n        self.patience = patience\n        self.verbose = verbose\n        self.delta = delta\n        self.best_value = None\n        self.best_cv = None\n        self.is_earlystop = False\n        self.count = 0\n        self.best_model = None\n        #self.val_preds = []\n        #self.val_logits = []\n\n    def earlystop(self, loss, value, model=None):#, preds, logits):\n        \"\"\"\n        value: evaluation value on valiation dataset\n        \"\"\"\n        cv = value\n        if self.best_value is None:\n            self.best_value = value\n            self.best_cv = cv\n            self.best_model = copy.deepcopy(model).to('cpu')\n            #self.val_preds = preds\n            #self.val_logits = logits\n        elif value < self.best_value + self.delta:\n            self.count += 1\n            if self.verbose:\n                print('EarlyStoper count: {:02d}'.format(self.count))\n            if self.count >= self.patience:\n                self.is_earlystop = True\n        else:\n            self.best_value = value\n            self.best_cv = cv\n            self.best_model = copy.deepcopy(model).to('cpu')\n            #self.val_preds = preds\n            #self.val_logits = logits\n            self.count = 0","metadata":{"execution":{"iopub.status.busy":"2022-06-07T03:37:06.480777Z","iopub.execute_input":"2022-06-07T03:37:06.484514Z","iopub.status.idle":"2022-06-07T03:37:06.501381Z","shell.execute_reply.started":"2022-06-07T03:37:06.484468Z","shell.execute_reply":"2022-06-07T03:37:06.500287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'model': 'LogisticRegression',\n    'batch_size': 2048,\n    'lr': [0.03, 0.01, 0.003],\n    'wd': [3e-4, 1e-4, 3e-5, 1e-5, 0.],\n    'repeat': 4,\n    #'device': 'cpu',\n    'device': 'cuda:0',\n    'early_stopping': 12,\n    'n_fold': 5,\n    'seed': 2021,\n    'max_epochs': 200,\n}\n\ndef binary_cross_entropy(pos_weight=None):\n    def weighted_bce(y_pred, y_true):\n        a = y_pred.reshape(-1)\n        b = y_true.reshape(-1)\n        if pos_weight is None:\n            weighted_loss =  (1-b) * torch.log(1-a) + b * torch.log(a)\n        elif isinstance(pos_weight, (list, torch.Tensor, np.ndarray)):\n            weighted_loss = (pos_weight[0] * (1-b) * torch.log(1-a) + pos_weight[1] * b * torch.log(a)) / sum(pos_weight)\n        elif isinstance(pos_weight, (int, float)):\n            weighted_loss = ((1-b) * torch.log(1-a) + pos_weight * b * torch.log(a)) / (pos_weight + 1)\n        else:\n            weighted_loss = 1\n        return torch.mean(weighted_loss)\n    return weighted_bce","metadata":{"execution":{"iopub.status.busy":"2022-06-07T03:37:06.50716Z","iopub.execute_input":"2022-06-07T03:37:06.510111Z","iopub.status.idle":"2022-06-07T03:37:06.526331Z","shell.execute_reply.started":"2022-06-07T03:37:06.510065Z","shell.execute_reply":"2022-06-07T03:37:06.525498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss_fn = nn.CrossEntropyLoss(weight=torch.from_numpy(np.array([118828, 340085])).float()).to(device)\n# loss_fn = nn.BCELoss(weight=torch.from_numpy(np.array([118828, 340085])))\n#for fold, (trn_idx, val_idx) in enumerate(kfold.split(train_nn, y_target)):\ndevice = params['device']\ntest_preds = list()\nfor fold in range(5):\n    print(f'Training fold {fold + 1}')\n    # x_train, x_val = num_feat.iloc[trn_idx], num_feat.iloc[val_idx]\n    x_train = pd.read_pickle(\"../input/amex-5-fold-agg-data/train_fold_{}.pkl\".format(fold))\n    x_val = pd.read_pickle(\"../input/amex-5-fold-agg-data/val_fold_{}.pkl\".format(fold))\n    print(\"Point 1!\")\n    features = [col for col in x_val.columns if col not in ['customer_ID', 'target']]\n    # scaler = StandardScaler()\n    x_train = torch.from_numpy(x_train[features].values).float().to(device)\n    x_val = torch.from_numpy(x_val[features].values).float().to(device)\n    print(\"Point 2!\")\n    y_train = pd.read_csv(\"../input/amex-5-fold-agg-data/train_fold_{}_target.csv\".format(fold))\n    pos_weight = torch.from_numpy((y_train.target * 1.862 + 1).values).float().to(device)\n    y_val = pd.read_csv(\"../input/amex-5-fold-agg-data/val_fold_{}_target.csv\".format(fold))\n    y_train = torch.from_numpy(y_train.target.values).float().to(device)\n    y_val = torch.from_numpy(y_val.target.values).float().to(device)\n    # loss_fn = nn.BCELoss(weight=torch.from_numpy(pos_weight.values))\n    train_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(y_train)),\n                                                                           num_samples=len(y_train), replacement=False)\n    train_dataloader = torch.utils.data.DataLoader(np.array(range(len(y_train))), batch_size=params['batch_size'], num_workers=0,\n                                                   sampler=train_sample_strategy, drop_last=False)\n    val_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(y_val)),\n                                                                         num_samples=len(y_val), replacement=False)\n    val_dataloader = torch.utils.data.DataLoader(np.array(range(len(y_val))), batch_size=params['batch_size'], num_workers=0,\n                                                 sampler=val_sample_strategy, drop_last=False)\n    model_list = [eval(params['model'])(x_train.shape[1], params['repeat']).to(device) for lr in params['lr'] for wd in params['wd']]\n    lr_list = [lr * np.sqrt(params['batch_size']/2048) for lr in params['lr'] for wd in params['wd']]\n    wd_list = [wd for lr in params['lr'] for wd in params['wd']]\n    optimizer_list = [optim.Adam(model.parameters(), lr=lr_list[i], weight_decay=wd_list[i]) for i, model in enumerate(model_list)]\n    lr_scheduler_list = [MultiStepLR(optimizer=optimizer, milestones=[800, 1600, 2400, 3200, 4000, 4800, 5600], gamma=0.6) for optimizer in optimizer_list]\n    earlystoper_list = [early_stopper(patience=params['early_stopping'], verbose=False) for lr in lr_list]\n    val_prediction_list = [torch.zeros(x_val.shape[0]).float().to(device) for lr in lr_list]\n    start_epoch = 0\n    print(\"Point 3!\")\n    for epoch in range(start_epoch, params['max_epochs']):\n        print(\"In epoch:{:03d}\".format(epoch))\n        train_loss_list = []\n        # train_acc_list = []\n        _ = [model.train() for model in model_list]\n        for step, input_seeds in enumerate(train_dataloader):\n            batch_inputs = x_train[input_seeds].to(device)\n            batch_labels = y_train[input_seeds].to(device)\n            weight = pos_weight[input_seeds].to(device)\n            for i, model in enumerate(model_list):\n                # model.train()\n                train_batch_logits = model(batch_inputs)\n                train_loss = F.binary_cross_entropy(train_batch_logits.mean(1), batch_labels, weight=weight)\n                # backward\n                optimizer_list[i].zero_grad()\n                train_loss.backward()\n                optimizer_list[i].step()\n                lr_scheduler_list[i].step()\n            # train_loss_list.append(train_loss.cpu().detach().numpy())\n            \n            # tr_batch_pred = None\n    \n        # mini-batch for validation\n        val_loss_list = 0\n        val_acc_list = 0\n        #val_correct_list = 0\n        val_all_list = 0\n        _ = [model.eval() for model in model_list]\n        with torch.no_grad():\n            for step, input_seeds in enumerate(val_dataloader):\n                batch_inputs = x_val[input_seeds].to(device)\n                batch_labels = y_val[input_seeds].to(device)\n                weight = pos_weight[input_seeds].to(device)\n                for i, model in enumerate(model_list):\n                    # model.eval()\n                    val_batch_logits = model(batch_inputs)\n                    val_prediction_list[i][input_seeds] = val_batch_logits.mean(1).detach()\n                # val_loss_list = val_loss_list + loss_fn(val_batch_logits, batch_labels)\n                # val_batch_pred = torch.sum(torch.argmax(val_batch_logits, dim=1) == batch_labels) / torch.tensor(batch_labels.shape[0])\n                # val_acc_list = val_acc_list + val_batch_pred * torch.tensor(batch_labels.shape[0])\n                # val_all_list = val_all_list + batch_labels.shape[0]\n            #tmp_predictions = model(test_feature).cpu().numpy()\n        #infold_preds[fold] = tmp_predictions\n        #test_predictions += tmp_predictions / params['n_fold']\n        semaphore = len(model_list)\n        for i, model in enumerate(model_list):\n            val_predictions = torch.sigmoid(val_prediction_list[i]).cpu().numpy()\n            earlystoper_list[i].earlystop(val_loss_list, amex_metric(y_val.float().cpu().numpy(), val_predictions), model)\n            if earlystoper_list[i].is_earlystop:\n                semaphore -= 1\n        if semaphore < 1:\n            print(\"Early Stopping\")\n            break\n                \n        # true = np.concatenate([true, y_valid])\n        # oof = np.concatenate([oof, val_predictions])\n        # if earlystoper.is_earlystop:\n            # print(\"Early Stopping!\")\n            # break\n    print(\"Best val_metric is:\\t\", \"\\t\".join([\"{:.4f}\".format(earlystoper.best_cv) for earlystoper in earlystoper_list]))\n    # print(\"Best val_metric is: {:.7f}\".format(earlystoper.best_cv))\n    x_test = pd.read_pickle(\"../input/amex-5-fold-agg-data/test_fold_{}.pkl\".format(fold))\n    # test_ids = x_test.customer_ID\n    test_prediction_list = [torch.zeros(len(x_test)) for i in lr_list]\n    test_sample_strategy = torch.utils.data.sampler.WeightedRandomSampler(np.ones(len(x_test)),\n                                                                          num_samples=len(x_test), replacement=False)\n    test_dataloader = torch.utils.data.DataLoader(np.array(range(len(x_test))), batch_size=params['batch_size'], num_workers=0,\n                                                  sampler=test_sample_strategy, drop_last=False)\n    x_test = torch.from_numpy(x_test[features].values).float().to(device)\n    test_predictions = torch.zeros(x_test.shape[0]).float()\n    # todo: start at this\n    # test_num_feat = torch.from_numpy(scaler.transform(test_nn[features_numerical])).float().to(device)\n    b_model_list = [earlystoper.best_model.to(device) for earlystoper in earlystoper_list]\n    _ = [b_model.eval() for b_model in b_model_list]\n    with torch.no_grad():\n        for step, input_seeds in enumerate(test_dataloader):\n            batch_inputs = x_test[input_seeds].to(device)\n            for i, b_model in enumerate(b_model_list):\n                # b_model.eval()\n                test_batch_logits = b_model(batch_inputs)\n                test_predictions[input_seeds] = test_predictions[input_seeds] + test_batch_logits.mean(1).detach().cpu()/len(b_model_list)\n            #test_batch_pred = torch.sum(torch.argmax(test_batch_logits, dim=1) == batch_labels) / torch.tensor(batch_labels.shape[0])\n            if step % 50 == 0:\n                print('In test batch:{:04d}'.format(step))\n    test_preds.append(test_predictions)\n    del x_train, x_val, x_test\ntest_preds = torch.stack(test_preds).mean(0).numpy()\n\n#my_acc = acc(y, oof_predictions)\n#my_ap = average_precision_score(y_target, torch.softmax(oof_predictions, dim=1).cpu()[:, 1])\n#print(\"NN out of fold AP is:\", my_ap)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-07T03:37:06.532702Z","iopub.execute_input":"2022-06-07T03:37:06.535751Z","iopub.status.idle":"2022-06-07T04:21:10.025217Z","shell.execute_reply.started":"2022-06-07T03:37:06.535703Z","shell.execute_reply":"2022-06-07T04:21:10.024124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nx_test = pd.read_pickle(\"../input/amex-5-fold-agg-data/test_fold_{}.pkl\".format(0))\nids = copy.deepcopy(x_test.customer_ID)\ndel x_test\ngc.collect()\nsub = pd.DataFrame()\nsub['customer_ID'] = ids\nsub['prediction'] = test_preds\nsub.to_csv(\"submission_LR_x_300.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T04:23:41.590016Z","iopub.execute_input":"2022-06-07T04:23:41.590432Z","iopub.status.idle":"2022-06-07T04:23:47.673212Z","shell.execute_reply.started":"2022-06-07T04:23:41.590399Z","shell.execute_reply":"2022-06-07T04:23:47.672314Z"},"trusted":true},"execution_count":null,"outputs":[]}]}