{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport pickle\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\n\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n    \nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom collections import Counter","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-09T12:58:55.732949Z","iopub.execute_input":"2022-08-09T12:58:55.733444Z","iopub.status.idle":"2022-08-09T12:58:55.740371Z","shell.execute_reply.started":"2022-08-09T12:58:55.733405Z","shell.execute_reply":"2022-08-09T12:58:55.739130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:59:02.016165Z","iopub.execute_input":"2022-08-09T12:59:02.016605Z","iopub.status.idle":"2022-08-09T12:59:02.023664Z","shell.execute_reply.started":"2022-08-09T12:59:02.016572Z","shell.execute_reply":"2022-08-09T12:59:02.022312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading dataset","metadata":{}},{"cell_type":"code","source":"%%time\ncustomer_ids = np.load(\"../input/amex-tabular-train-nn-dataset-v3/train_data/customer_ids.npy\")\nx_train = np.load(\"../input/amex-tabular-train-nn-dataset-v3/train_data/x_train.npy\")\ny_train = np.load(\"../input/amex-tabular-train-nn-dataset-v3/train_data/y_train.npy\")\nx_miss_train = np.load(\"../input/amex-tabular-train-nn-dataset-v3/train_data/x_miss_train.npy\")\nx_count = np.load(\"../input/amex-tabular-train-nn-dataset-v3/train_data/x_count.npy\")\n\nx_train = np.clip(x_train, -3.0, 3.0)\n\nprint(x_train.shape, x_count.shape, x_miss_train.shape, y_train.shape)\nprint()\nprint(x_train.dtype)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:59:05.062414Z","iopub.execute_input":"2022-08-09T12:59:05.062824Z","iopub.status.idle":"2022-08-09T12:59:23.321033Z","shell.execute_reply.started":"2022-08-09T12:59:05.062789Z","shell.execute_reply":"2022-08-09T12:59:23.319933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    BATCH_SIZE = 4096\n    N_EPOCHS=12","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:59:23.323099Z","iopub.execute_input":"2022-08-09T12:59:23.323481Z","iopub.status.idle":"2022-08-09T12:59:23.328995Z","shell.execute_reply.started":"2022-08-09T12:59:23.323449Z","shell.execute_reply":"2022-08-09T12:59:23.327823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class AmexDataset(torch.utils.data.Dataset):\n    def __init__(self, X, xcount, xmiss_count, y, idxs, phase='train'):\n        self.idxs=idxs\n        self.X = X\n        self.xcount = xcount\n        self.xmiss_count = xmiss_count\n        self.y = y\n        self.phase=phase\n    \n    def __getitem__(self, idx):\n        idx = self.idxs[idx]\n        x = torch.tensor(self.X[idx], dtype=torch.float32)\n        xmissing = 1+(x==-1).type(torch.long)\n        \n        x_count = torch.tensor(self.xcount[idx]/13, dtype=torch.float32)\n        xmiss_count = torch.tensor(self.xmiss_count[idx]/13, dtype=torch.float32)\n        \n        \n        if self.phase == 'train':\n            y = torch.tensor(self.y[idx], dtype=torch.float32)\n            return (x, xmissing, x_count, xmiss_count, y)\n        return x, xmissing, x_count, xmiss_count\n    \n    def __len__(self):\n        return len(self.idxs)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:59:23.330767Z","iopub.execute_input":"2022-08-09T12:59:23.331107Z","iopub.status.idle":"2022-08-09T12:59:23.343577Z","shell.execute_reply.started":"2022-08-09T12:59:23.331076Z","shell.execute_reply":"2022-08-09T12:59:23.342012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"code","source":"class TransformBlock(nn.Module):\n    def __init__(self, insize, outsize, dropout=0.1):\n        super().__init__()\n        self.bn = nn.BatchNorm1d(insize)\n        self.linear = nn.Linear(insize, outsize)\n        self.activation = nn.Softplus()\n        self.dropout = nn.Dropout(dropout)\n    \n    def forward(self, x):\n        x = self.bn(x)\n        x = self.linear(x)\n        x = self.activation(x)\n        x = self.dropout(x)\n        return x\n    \nclass TabularModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.pre_bn = nn.BatchNorm1d(1401)\n        self.missing_embedd = nn.Embedding(3, 5)\n        self.preprocess_layer = nn.Linear(6, 1)\n        self.dropout=nn.Dropout(0.01)\n        \n        self.layer1 = TransformBlock(1401, 1024, dropout = 0.5)\n        self.layer2 = TransformBlock(1024, 512, dropout = 0.5)\n        self.layer3 = TransformBlock(512, 256, dropout = 0.5)\n        \n        self.head = nn.Sequential(\n            nn.Linear(256, 1)\n        )\n        \n    def forward(self, x, xmissing, x_count, xmiss_count):\n        xmissing = self.missing_embedd(xmissing)\n        x = x.unsqueeze(dim=-1)\n        x = torch.cat([x, xmissing], dim=-1)\n        \n        xmeta = torch.cat([x_count.unsqueeze(dim=-1), xmiss_count], dim=-1)\n        x = self.preprocess_layer(x).squeeze(dim=-1)\n        x = torch.cat([x, xmeta], dim=-1)\n        \n        x = self.dropout(x)\n        \n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        \n        y = self.head(x).view(-1)\n        return {'y': y}","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.821631Z","iopub.execute_input":"2022-08-09T13:00:48.822144Z","iopub.status.idle":"2022-08-09T13:00:48.839621Z","shell.execute_reply.started":"2022-08-09T13:00:48.822105Z","shell.execute_reply":"2022-08-09T13:00:48.838252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# metrics","metadata":{}},{"cell_type":"code","source":"def top_4percent(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    four_percent_cutoff = 0.04 * sum(df['weight'])\n    df['weight_cumsum'] = df['weight'].cumsum()\n    df_cutoff = df[df.weight_cumsum <= four_percent_cutoff]\n    \n    return df_cutoff['target'].sum()/df['target'].sum()\n\ndef weighted_gini(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n    total_pos = (df['target'] * df['weight']).sum()\n    df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n    df['lorentz'] = df['cum_pos_found'] / total_pos\n    df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n    return df['gini'].sum()\n\n\ndef normalized_gini(df):\n    df_true=df[['target']].copy()\n    df_true['pred'] = df_true['target'].copy()\n    \n    G = weighted_gini(df)/weighted_gini(df_true)\n    return G","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.841729Z","iopub.execute_input":"2022-08-09T13:00:48.842711Z","iopub.status.idle":"2022-08-09T13:00:48.856832Z","shell.execute_reply.started":"2022-08-09T13:00:48.842665Z","shell.execute_reply":"2022-08-09T13:00:48.855693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(foldnum, val_index, model, val_dataloader):\n    model.eval()\n    ytrue=[]\n    ypred=[]\n    \n    for (x, xmissing, x_count, xmiss_count, y) in val_dataloader:\n        x = x.to(device)\n        xmissing = xmissing.to(device)\n        x_count = x_count.to(device)\n        xmiss_count = xmiss_count.to(device)\n        y = y.to(device)\n        \n        with torch.no_grad():\n            outputs=model(x, xmissing, x_count, xmiss_count)\n            yhat = outputs['y']\n            ytrue += y.cpu().tolist()\n            ypred += yhat.cpu().tolist()\n    \n    df = pd.DataFrame.from_dict({\n        'customer_ids': customer_ids[val_index],\n        'target': ytrue,\n        'pred': ypred\n    })\n    df['predlabel'] = (df['pred'] > 0.5).astype(int)\n    \n    \n    \n    ypred0 = (df[df.target==0].pred).mean()\n    ypred1 = (df[df.target==1].pred).mean()\n    \n    print()\n    print()\n    print()\n    print(\"avg non-defaulter prob:{:.4f}\".format(ypred0))\n    print(\"avg defaulter prob:{:.4f}\".format(ypred1))\n    \n    print(\"f1_score:{:.4f}\".format(f1_score(df.target, df.predlabel)))\n    print(\"proportion of non defaulter >0.5: {:.4f}\".format(len(df[(df.target==0) & (df.pred>=0.5)])/len(df) ))\n    print(\"proportion of defaulter < 0.5: {:.4f}\".format(len(df[(df.target==1) & (df.pred <= 0.5)])/len(df) ))\n    print()\n    print()\n    print()\n    \n    \n    df.to_csv(\"eval_{}.csv\".format(foldnum), index=False)\n    G = normalized_gini(df[['target', 'pred']])\n    D = top_4percent(df[['target', 'pred']])\n    M = (G+D)/2\n    \n    return (G, D, M)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.864315Z","iopub.execute_input":"2022-08-09T13:00:48.865067Z","iopub.status.idle":"2022-08-09T13:00:48.880945Z","shell.execute_reply.started":"2022-08-09T13:00:48.865015Z","shell.execute_reply":"2022-08-09T13:00:48.880023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train model","metadata":{}},{"cell_type":"code","source":"def get_rank_loss(yhat, y):\n    loss = torch.tensor(0.0, device=device)\n    ypos = yhat[y==1]\n    yneg = yhat[y==0]\n    \n    if len(ypos) == 0 or len(yneg) == 0:\n        return loss\n    \n    yneg = yneg.repeat((len(ypos), 1))\n    ypos = ypos.unsqueeze(dim=-1)\n    loss1 = -torch.log( torch.sigmoid( ypos.detach()-yneg) ).mean()\n    loss2 = -torch.log( torch.sigmoid( ypos-yneg.detach()) ).mean()\n    loss = (loss1+loss2)/2\n    \n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.889850Z","iopub.execute_input":"2022-08-09T13:00:48.890323Z","iopub.status.idle":"2022-08-09T13:00:48.899916Z","shell.execute_reply.started":"2022-08-09T13:00:48.890282Z","shell.execute_reply":"2022-08-09T13:00:48.898787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hinge_loss(yhat, y):\n    yhat = torch.clamp(yhat, -3, 3)\n    yerr = y*(1 - yhat) + (1-y) * (1+yhat)\n    yerr = torch.clamp(yerr, 0, 3)\n    loss = torch.mean(yerr)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.917913Z","iopub.execute_input":"2022-08-09T13:00:48.918673Z","iopub.status.idle":"2022-08-09T13:00:48.927404Z","shell.execute_reply.started":"2022-08-09T13:00:48.918623Z","shell.execute_reply":"2022-08-09T13:00:48.925967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_ops(X, xmissing, x_count, xmiss_count, y, criterion, model, optimizer, scheduler):\n    X = X.to(device)\n    xmissing = xmissing.to(device)\n    x_count=x_count.to(device)\n    xmiss_count = xmiss_count.to(device)\n    y = y.to(device)\n    \n    model.train()\n    outputs = model(X, xmissing, x_count, xmiss_count)\n    yhat = outputs['y']\n    \n    binary_loss = criterion(yhat, y)\n    hinge_loss = get_hinge_loss(yhat, y)\n    rank_loss = get_rank_loss(yhat, y)\n    loss = binary_loss + hinge_loss + rank_loss\n    \n    \n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    torch.nn.utils.clip_grad_norm_(model.parameters(), 5)\n    optimizer.step()\n    scheduler.step()\n    \n    losses = {\n        'loss': loss.item(),\n        'binary_loss': binary_loss.item(),\n        'hinge_loss': hinge_loss.item(),\n        'rank_loss': rank_loss.item()\n    }\n    return losses\n\ndef train_fold(s, foldnum, val_index, train_dataloader, val_dataloader):\n    model = TabularModel().to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-3)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, \n                                                           T_max = CFG.N_EPOCHS * len(train_dataloader), \n                                                           eta_min=1e-7)\n    \n    best_eval=None\n    evals=[]\n    for e in range(CFG.N_EPOCHS):\n        epoch_loss=[]\n        epoch_binary_loss=[]\n        epoch_hinge_loss=[]\n        epoch_rank_loss=[]\n        \n        for it, (X, xmissing, x_count, xmiss_count, y) in enumerate(train_dataloader):\n            losses = train_ops(X, xmissing, x_count, xmiss_count, y, criterion, model, optimizer, scheduler)\n            loss = losses['loss']\n            hinge_loss = losses['hinge_loss']\n            rank_loss = losses['rank_loss']\n            binary_loss = losses['binary_loss']\n            \n            epoch_loss.append(loss)\n            epoch_binary_loss.append(binary_loss)\n            epoch_hinge_loss.append(hinge_loss)\n            epoch_rank_loss.append(rank_loss)\n        \n        (G, D, M) = evaluate(foldnum, val_index, model, val_dataloader)\n        if best_eval is None or best_eval < M:\n            best_eval = M\n            torch.save(model, \"models/tabular_model_{}_{}.pt\".format(s, foldnum))\n            \n        evals.append(M)\n        print(\"epoch:{} | train loss:{:.4f} | rankloss: {:.4f}\".format(e, np.mean(epoch_loss), np.mean(epoch_rank_loss)))\n        print(\"hinge loss:{:.4f}\".format(np.mean(epoch_hinge_loss)))\n        print(\"binary loss:{:.4f}\".format(np.mean(epoch_binary_loss)))\n        print(\"Eval:{:.6f} | best eval:{:.6f}\".format(M, best_eval))\n        print(\"Gini:{:.6f} | Default Rate:{:.6f}\".format(G, D))\n    \n    print(\"End of training foldnumber:\", foldnum)\n    print(\"Best Eval:\",best_eval)\n    plt.title(\"evals...\")\n    plt.plot(evals)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.931398Z","iopub.execute_input":"2022-08-09T13:00:48.932503Z","iopub.status.idle":"2022-08-09T13:00:48.952641Z","shell.execute_reply.started":"2022-08-09T13:00:48.932451Z","shell.execute_reply":"2022-08-09T13:00:48.951714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(\"models\"):\n    os.mkdir(\"models\")","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.955181Z","iopub.execute_input":"2022-08-09T13:00:48.955666Z","iopub.status.idle":"2022-08-09T13:00:48.968105Z","shell.execute_reply.started":"2022-08-09T13:00:48.955630Z","shell.execute_reply":"2022-08-09T13:00:48.966833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=88471, shuffle=True)\nfor foldnum, (train_index, val_index) in enumerate(skf.split(y_train, y_train)):\n    train_dataset = AmexDataset(x_train, x_count, x_miss_train, y_train, train_index)\n    val_dataset = AmexDataset(x_train, x_count, x_miss_train, y_train, val_index)\n    \n    train_dataloader = torch.utils.data.DataLoader(train_dataset, \n                                                   batch_size=CFG.BATCH_SIZE, \n                                                   shuffle=True)\n    \n    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE, \n                                                 shuffle=False,\n                                                 drop_last=False)\n    \n    \n    print(\"====================================================\")\n    print(\"Foldnumber:\", foldnum)\n    print(\"number of train batches:\", len(train_dataloader))\n    print(\"number of val batches:\", len(val_dataloader))\n    \n    for s in range(1):\n        train_fold(s, foldnum, val_index, train_dataloader, val_dataloader)\n    gc.collect()\n    print()\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:00:48.977125Z","iopub.execute_input":"2022-08-09T13:00:48.977769Z","iopub.status.idle":"2022-08-09T13:00:55.538105Z","shell.execute_reply.started":"2022-08-09T13:00:48.977732Z","shell.execute_reply":"2022-08-09T13:00:55.536076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}