{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport pickle\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\n\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-22T15:38:51.258756Z","iopub.execute_input":"2022-08-22T15:38:51.259312Z","iopub.status.idle":"2022-08-22T15:38:53.653657Z","shell.execute_reply.started":"2022-08-22T15:38:51.259202Z","shell.execute_reply":"2022-08-22T15:38:53.652354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    BATCH_SIZE=2048\n    N_EPOCHS=12","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:38:53.656376Z","iopub.execute_input":"2022-08-22T15:38:53.656972Z","iopub.status.idle":"2022-08-22T15:38:53.663132Z","shell.execute_reply.started":"2022-08-22T15:38:53.656934Z","shell.execute_reply":"2022-08-22T15:38:53.661891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:38:53.665047Z","iopub.execute_input":"2022-08-22T15:38:53.666204Z","iopub.status.idle":"2022-08-22T15:38:53.681793Z","shell.execute_reply.started":"2022-08-22T15:38:53.666153Z","shell.execute_reply":"2022-08-22T15:38:53.680431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading the dataset","metadata":{}},{"cell_type":"code","source":"%%time\nall_train_X = np.load(\"../input/amex-sequence-dataset-v3/train_sequence/all_train_num_feats.npy\")\nall_train_X = np.clip(all_train_X, -6, 6)\n\nall_train_cat = np.load(\"../input/amex-sequence-dataset-v3/train_sequence/all_train_cat_feats.npy\")\nall_seqlens = np.load(\"../input/amex-sequence-dataset-v3/train_sequence/all_seqlens.npy\")\n\nall_train_labels = np.load(\"../input/amex-sequence-dataset-v3/train_sequence/all_targets.npy\")\nprint(all_train_X.shape, all_train_cat.shape, all_train_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:38:53.684214Z","iopub.execute_input":"2022-08-22T15:38:53.685337Z","iopub.status.idle":"2022-08-22T15:39:51.650518Z","shell.execute_reply.started":"2022-08-22T15:38:53.685277Z","shell.execute_reply":"2022-08-22T15:39:51.648995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(all_train_X.max(), all_train_X.min())","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:39:51.655039Z","iopub.execute_input":"2022-08-22T15:39:51.655755Z","iopub.status.idle":"2022-08-22T15:39:52.503835Z","shell.execute_reply.started":"2022-08-22T15:39:51.655706Z","shell.execute_reply":"2022-08-22T15:39:52.502569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_seqlens[:4]","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:39:52.506274Z","iopub.execute_input":"2022-08-22T15:39:52.506692Z","iopub.status.idle":"2022-08-22T15:39:52.516896Z","shell.execute_reply.started":"2022-08-22T15:39:52.506656Z","shell.execute_reply":"2022-08-22T15:39:52.515522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_customer_ids = np.load(\"../input/amex-sequence-dataset-v3/train_sequence/all_train_customerIds.npy\")\ntrain_customer_ids.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:39:52.518948Z","iopub.execute_input":"2022-08-22T15:39:52.519868Z","iopub.status.idle":"2022-08-22T15:39:52.577304Z","shell.execute_reply.started":"2022-08-22T15:39:52.519814Z","shell.execute_reply":"2022-08-22T15:39:52.576044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dataset","metadata":{}},{"cell_type":"code","source":"class AmexDataset(torch.utils.data.Dataset):\n    def __init__(self, X, Xcat, y, idxs, allseqlens=None, phase='train'):\n        self.idxs=idxs\n        self.X = X\n        self.Xcat=Xcat\n        self.y = y\n        self.allseqlens=allseqlens\n        self.phase=phase\n    \n    def __getitem__(self, idx):\n        idx=self.idxs[idx]\n        xnum = torch.tensor(self.X[idx], dtype=torch.float32)\n        xmissing = 1+(xnum==-1).type(torch.long)\n        xcat = torch.tensor(self.Xcat[idx], dtype=torch.long)\n        seqlen = self.allseqlens[idx]\n        \n        mask = torch.ones(13, dtype=torch.long)\n        mask[-seqlen:]=0\n        \n        if self.phase !='train':\n            return xnum, xmissing, xcat, mask\n        \n        y = torch.tensor(self.y[idx], dtype=torch.float32)\n        return (xnum, xmissing, xcat,mask, y)\n    \n    def __len__(self):\n        return len(self.idxs)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:39:52.578923Z","iopub.execute_input":"2022-08-22T15:39:52.579347Z","iopub.status.idle":"2022-08-22T15:39:52.591313Z","shell.execute_reply.started":"2022-08-22T15:39:52.579311Z","shell.execute_reply":"2022-08-22T15:39:52.590026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class TransformerProjectionBlock(nn.Module):\n    def __init__(self, in_size, out_size):\n        super().__init__()\n        self.dropout = nn.Dropout(0.5)\n        self.bn = nn.BatchNorm1d(in_size)\n        self.linear = nn.Linear(in_size, out_size)\n        self.activation = nn.Softplus()\n        \n        \n    def forward(self, x):\n        x = self.dropout(x)\n        x = self.bn(x)\n        x = self.linear(x)\n        x = self.activation(x)\n        return x\n\nclass TransformerProjectionMLP(nn.Module):\n    def __init__(self, sz):\n        super().__init__()\n        self.proj1 = TransformerProjectionBlock(sz, sz//2)\n        self.proj2 = TransformerProjectionBlock(sz//2, sz//4)\n        self.attn_proj = nn.Sequential(\n            nn.Linear(13, 48),\n            nn.Softplus(),\n            nn.Dropout(0.5),\n            nn.Linear(48, 13),\n            nn.Softplus(),\n            nn.Dropout(0.3)\n        )\n        \n        \n        self.dropout = nn.Dropout(0.3)\n        self.bn = nn.BatchNorm1d(sz//4)\n        \n        self.out = nn.Linear(13+(sz//4), 1)\n        \n        \n        self.proj1.linear.weight.data.uniform_(-0.03, 0.03)\n        self.proj1.linear.bias.data.uniform_(-0.03, 0.03)\n        \n        self.proj2.linear.weight.data.uniform_(-0.03, 0.03)\n        self.proj2.linear.bias.data.uniform_(-0.03, 0.03)\n        \n    def forward(self, x, x_attn_weights):\n        x = self.proj1(x)\n        x = self.proj2(x)\n        x = self.dropout(x)\n        x = self.bn(x)\n        \n        x_attn_weights = self.attn_proj(x_attn_weights)\n        x = self.out( torch.cat([x, x_attn_weights], dim=-1 ) )\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:39:42.967285Z","iopub.execute_input":"2022-08-22T16:39:42.968162Z","iopub.status.idle":"2022-08-22T16:39:42.983823Z","shell.execute_reply.started":"2022-08-22T16:39:42.968111Z","shell.execute_reply":"2022-08-22T16:39:42.982366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformBlock(nn.Module):\n    def __init__(self, insize, outsize):\n        super().__init__()\n        self.W = nn.Linear(insize, 2*outsize)\n        self.activation = nn.GLU()\n        self.dropout = nn.Dropout(0.2)\n    def forward(self, x):\n        x = self.W(x)\n        x = self.activation(x)\n        x = self.dropout(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:23:19.411109Z","iopub.execute_input":"2022-08-22T16:23:19.411549Z","iopub.status.idle":"2022-08-22T16:23:19.418668Z","shell.execute_reply.started":"2022-08-22T16:23:19.411512Z","shell.execute_reply":"2022-08-22T16:23:19.417780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiheadAttention(nn.Module):\n    def __init__(self, dmodel, nhead):\n        super().__init__()\n        self.ln = nn.LayerNorm(dmodel)\n        self.multiheadAttention = nn.MultiheadAttention(dmodel, nhead, dropout=0.1, batch_first=True)\n        self.dropout = nn.Dropout(0.2)\n        self.activation = nn.GELU()\n        \n        self.ffn = nn.Sequential(\n            nn.Linear(dmodel, 2*dmodel),\n            nn.Softplus(),\n            nn.Dropout(0.2),\n            nn.Linear(2*dmodel, dmodel),\n        )\n        \n    def forward(self, x, mask):\n        x = self.ln(x)\n        x1, attn_weights = self.multiheadAttention(x, x, x, key_padding_mask = mask)\n        x1 = self.activation(x1)\n        x2 = x + self.dropout(x1)\n        x = x2+self.ffn(x2)\n        \n        return x, attn_weights","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.606546Z","iopub.execute_input":"2022-08-22T16:24:28.607040Z","iopub.status.idle":"2022-08-22T16:24:28.617262Z","shell.execute_reply.started":"2022-08-22T16:24:28.607003Z","shell.execute_reply":"2022-08-22T16:24:28.616152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AmexTransformerModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        DIM = 256\n        self.embeddings = nn.ModuleList([nn.Embedding(10, 4, padding_idx=0) for _ in range(13)])    \n        self.time_embedding = nn.Embedding(13, DIM)\n        self.missing_embedd = nn.Embedding(3, 5)\n        self.preprocess_layer = nn.Linear(6, 1)\n        \n        self.pre_ln = nn.LayerNorm(175)\n        self.layernorm1 = nn.LayerNorm(DIM)\n        self.layernorm2 = nn.LayerNorm(DIM)\n        \n        self.transform_block1 = TransformBlock(175+52, DIM)\n        self.transform_block2 = TransformBlock(DIM, DIM)\n        \n        self.attention1 = MultiheadAttention(DIM, 16)\n        self.attention2 = MultiheadAttention(DIM, 16)\n        \n        self.attn_mask_embedding = nn.Embedding(3, 5)\n        self.attn_mask_processing = nn.Linear(6, 1)\n        \n        self.mlp = TransformerProjectionMLP(DIM)\n    \n    def precomputewithmissingvalues(self, x, xmissing):\n        x_list = []\n        for i in range(13):\n            xi = x[:, i].unsqueeze(dim=-1)\n            xmissing_i = self.missing_embedd(xmissing[:, i])\n            xi = torch.cat([xi, xmissing_i], dim=-1)\n            xi = self.preprocess_layer(xi).squeeze(dim=-1).unsqueeze(dim=1)\n            x_list.append(xi)\n        x = torch.cat(x_list, dim=1)\n        return x\n    \n    def concat_categorical_embeddings(self, x, xcat):\n        xcat_embedds = []\n        for i in range(13):\n            xcat_embedds.append( self.embeddings[i](xcat[:, :, i]) )\n        xcat_embedds = torch.cat(xcat_embedds, dim=-1)\n        x = torch.cat([x, xcat_embedds], dim=-1)\n        return x\n    \n    \n    def get_mean(self, x, mask):\n        xsum = (x * mask.unsqueeze(dim=-1)).sum(dim=1)\n        seqlens = (13-mask.sum(dim=-1)).unsqueeze(dim=-1)\n        xmean = xsum * seqlens\n        return xmean\n    \n    def forward(self, x, xmissing, xcat, mask):\n        batch_size = len(x)\n        xtime = self.time_embedding(torch.arange(12, -1, -1).repeat(batch_size).view(-1, 13).to(device))\n        x = self.precomputewithmissingvalues(x, xmissing)\n        \n        x = self.pre_ln(x)\n        x = self.concat_categorical_embeddings(x, xcat)\n        \n        x = self.layernorm1(self.transform_block1(x))\n        x = self.transform_block2(x)\n        \n        x, attn_weights = self.attention1(x+xtime, mask)\n        x, attn_weights = self.attention2(x, mask)\n        \n        \n        x_attn_weights = attn_weights[:,-1,:]\n        x_attn_weights = x_attn_weights.masked_fill((1-mask).type(torch.bool), -1.0)\n        x_attn_weights = torch.cat([x_attn_weights.unsqueeze(dim=-1), self.attn_mask_embedding(mask)], dim=-1)\n        x_attn_weights = self.attn_mask_processing(x_attn_weights).squeeze(dim=-1)\n        \n        xlast = x[:, -1, :]\n        #xmean = self.get_mean(x, mask)\n        #xlag = xlast - xmean\n        #x, attn_weights = self.attention2(x)\n        #attn_weights = attn_weights[:, -3:, :]\n        #attn_weights = attn_weights.reshape(batch_size, -1)\n        #x = torch.cat([xlast, x_attn_weights], dim=-1)\n        \n        y = self.mlp(xlast, x_attn_weights).view(-1)\n        return {\n            'y': y,\n            'mean_attn_weights': np.zeros(13)#attn_weights.mean(dim=0).detach().cpu().numpy()\n        }","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.621378Z","iopub.execute_input":"2022-08-22T16:24:28.621957Z","iopub.status.idle":"2022-08-22T16:24:28.644326Z","shell.execute_reply.started":"2022-08-22T16:24:28.621919Z","shell.execute_reply":"2022-08-22T16:24:28.643423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"def top_4percent(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    four_percent_cutoff = 0.04 * sum(df['weight'])\n    df['weight_cumsum'] = df['weight'].cumsum()\n    df_cutoff = df[df.weight_cumsum <= four_percent_cutoff]\n    \n    return df_cutoff['target'].sum()/df['target'].sum()\n\ndef weighted_gini(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n    total_pos = (df['target'] * df['weight']).sum()\n    df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n    df['lorentz'] = df['cum_pos_found'] / total_pos\n    df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n    return df['gini'].sum()\n\ndef normalized_gini(df):\n    df_true=df[['target']].copy()\n    df_true['pred'] = df_true['target'].copy()\n    \n    G = weighted_gini(df)/weighted_gini(df_true)\n    return G","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.646737Z","iopub.execute_input":"2022-08-22T16:24:28.647889Z","iopub.status.idle":"2022-08-22T16:24:28.662751Z","shell.execute_reply.started":"2022-08-22T16:24:28.647838Z","shell.execute_reply":"2022-08-22T16:24:28.661597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train model","metadata":{}},{"cell_type":"code","source":"def get_rank_loss(yhat, y):\n    loss = torch.tensor(0.0, device=device)\n    ypos = yhat[y==1]\n    yneg = yhat[y==0]\n    \n    if len(ypos) == 0 or len(yneg) == 0:\n        return loss\n    \n    yneg = yneg.repeat((len(ypos), 1))\n    ypos = ypos.unsqueeze(dim=-1)\n    loss1 = -torch.log( 1e-8 + torch.sigmoid( ypos.detach()-yneg) ).mean()\n    loss2 = -torch.log( 1e-8 + torch.sigmoid( ypos-yneg.detach()) ).mean()\n    \n    loss = (loss1+loss2)/2\n    return loss\n\ndef get_hinge_loss(yhat, y):\n    yhat = torch.clamp(yhat, -3, 3)\n    yerr = y*(1 - yhat) + (1-y) * (1+yhat)\n    yerr = torch.clamp(yerr, 1e-8, 3)\n    loss = torch.mean(yerr)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.664843Z","iopub.execute_input":"2022-08-22T16:24:28.665313Z","iopub.status.idle":"2022-08-22T16:24:28.680576Z","shell.execute_reply.started":"2022-08-22T16:24:28.665267Z","shell.execute_reply":"2022-08-22T16:24:28.679660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(s, foldnum, val_index, model, val_dataloader):\n    model.eval()\n    ytrue=[]\n    ypred=[]\n    \n    for (xnum, xmissing, xcat, mask, y) in val_dataloader:\n        xnum = xnum.to(device)\n        xmissing = xmissing.to(device)\n        xcat = xcat.to(device)\n        mask = mask.to(device)\n        y = y.to(device)\n        \n        with torch.no_grad():\n            outputs=model(xnum, xmissing, xcat, mask)\n            yhat = outputs['y'].sigmoid()\n            ytrue += y.cpu().tolist()\n            ypred += yhat.cpu().tolist()\n    \n    df = pd.DataFrame.from_dict({\n        'customer_ids': train_customer_ids[val_index],\n        'target': ytrue,\n        'pred': ypred\n    })\n    df['predlabel'] = (df['pred'] > 0.5).astype(int)\n    \n    \n    print(\"====================================================\")\n    ypred0 = (df[df.target==0].pred).mean()\n    ypred1 = (df[df.target==1].pred).mean()\n    \n    print(\"avg non-defaulter prob:{:.4f}\".format(ypred0))\n    print(\"avg defaulter prob:{:.4f}\".format(ypred1))\n    \n    print(\"f1_score:{:.4f}\".format(f1_score(df.target, df.predlabel)))\n    print(\"proportion of non defaulter >0.3: {:.4f}\".format(len(df[(df.target==0) & (df.pred>=0.3)])/len(df) ))\n    print(\"proportion of defaulter < 0.7: {:.4f}\".format(len(df[(df.target==1) & (df.pred <= 0.7)])/len(df) ))\n    print(\"====================================================\")\n    print()\n    \n    \n    df.to_csv(\"gru_{}_{}.csv\".format(s, foldnum), index=False)\n    G = normalized_gini(df[['target', 'pred']])\n    D = top_4percent(df[['target', 'pred']])\n    M = (G+D)/2\n    \n    return (G, D, M)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.681977Z","iopub.execute_input":"2022-08-22T16:24:28.682932Z","iopub.status.idle":"2022-08-22T16:24:28.697485Z","shell.execute_reply.started":"2022-08-22T16:24:28.682895Z","shell.execute_reply":"2022-08-22T16:24:28.696298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_losses(y, yhat, criterion):\n    binary_loss  = criterion(yhat, y)\n    rank_loss  = get_rank_loss(yhat, y)\n    hinge_loss = get_hinge_loss(yhat, y)\n    \n    loss = binary_loss + rank_loss+ hinge_loss\n    \n    return (loss, binary_loss, rank_loss, hinge_loss)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.700416Z","iopub.execute_input":"2022-08-22T16:24:28.700887Z","iopub.status.idle":"2022-08-22T16:24:28.714245Z","shell.execute_reply.started":"2022-08-22T16:24:28.700846Z","shell.execute_reply":"2022-08-22T16:24:28.713093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(s, foldnum, val_index, train_dataloader, val_dataloader):\n    best_eval=None\n    \n    model = AmexTransformerModel().to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    mse_loss = nn.MSELoss()\n    \n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-3)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, \n                                                           T_max = CFG.N_EPOCHS * len(train_dataloader), \n                                                           eta_min=1e-6)\n    \n    for e in range(CFG.N_EPOCHS):\n        epoch_loss=[]\n        epoch_loss_embedd=[]\n        epoch_loss_embedd0=[]\n        epoch_loss_embedd1=[]\n        \n        epoch_binary_loss=[]\n        epoch_rank=[]\n        epoch_hinge_loss=[]\n        \n        mean_attn_weights = np.zeros(13)\n        \n        model.train()\n        for it, (xnum, xmissing, xcat,mask, y) in enumerate(train_dataloader):\n            xnum = xnum.to(device)\n            xmissing = xmissing.to(device)\n            xcat = xcat.to(device)\n            mask = mask.to(device)\n            y = y.to(device)\n            \n            outputs = model(xnum, xmissing, xcat, mask)\n            (loss, binary_loss, rank_loss, hinge_loss) = get_losses(y, outputs['y'], criterion)\n            \n            optimizer.zero_grad(set_to_none=True)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 5)\n            optimizer.step()\n            scheduler.step()\n\n            epoch_loss.append(loss.item()) \n            epoch_binary_loss.append(binary_loss.item())\n            epoch_rank.append(rank_loss.item())\n            epoch_hinge_loss.append(hinge_loss.item())\n            \n            mean_attn_weights+=outputs['mean_attn_weights']\n            \n        #Evaluating\n        (G, D, M) = evaluate(s, foldnum, val_index, model, val_dataloader)\n        if best_eval is None or best_eval<M:\n            best_eval = M\n            torch.save(model, \"models/model_{}_{}.pt\".format(s, foldnum))\n            \n        \n        print(\"epoch:{} | loss:{:.4f} | rank:{:.4f}\".format(e, np.mean(epoch_loss), np.mean(epoch_rank)))\n        print(\"binary loss:{:.4f}\".format(np.mean(epoch_binary_loss)))\n        print(\"hinge loss:{:.4f}\".format(np.mean(epoch_hinge_loss)))\n        print(\"current Eval:{:.4f} | best Eval:{:.4f}\".format(M, best_eval))\n        print(\"Gini:{:.4f} | Default Rate:{:4f}\".format(G, D))\n        print()\n        print()\n        print(\"mean attention weights:\")\n        print(mean_attn_weights/len(train_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.715649Z","iopub.execute_input":"2022-08-22T16:24:28.716057Z","iopub.status.idle":"2022-08-22T16:24:28.734156Z","shell.execute_reply.started":"2022-08-22T16:24:28.716021Z","shell.execute_reply":"2022-08-22T16:24:28.732926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('models'):\n    os.mkdir(\"models\")","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.736641Z","iopub.execute_input":"2022-08-22T16:24:28.738936Z","iopub.status.idle":"2022-08-22T16:24:28.746824Z","shell.execute_reply.started":"2022-08-22T16:24:28.738882Z","shell.execute_reply":"2022-08-22T16:24:28.745477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=88471, shuffle=True)\nfor foldnum, (train_index, val_index) in enumerate(skf.split(all_train_labels, all_train_labels)):\n    train_dataset = AmexDataset(all_train_X, all_train_cat, all_train_labels, train_index, allseqlens=all_seqlens)\n    val_dataset = AmexDataset(all_train_X, all_train_cat, all_train_labels, val_index, allseqlens=all_seqlens)\n    \n    train_dataloader = torch.utils.data.DataLoader(train_dataset, \n                                                   batch_size=CFG.BATCH_SIZE, \n                                                   shuffle=True,\n                                                   drop_last=True)\n    \n    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE, \n                                                   shuffle=False,\n                                                   drop_last=False)\n    \n    \n    for s in range(1):\n        print(\"Foldnumber:\", foldnum)\n        print(\"Seed:\", s)\n        print(\"number of train iterations:\", len(train_dataloader))\n        print(\"number of val iterations:\", len(val_dataloader))\n\n        train_model(s, foldnum, val_index, train_dataloader, val_dataloader)\n\n        #To save the eval csv\n        model = torch.load(\"models/model_{}_{}.pt\".format(s, foldnum), map_location=device)\n        (G, D, M) = evaluate(s, foldnum, val_index, model, val_dataloader)\n        print(\"***************End of foldnumber{} - Seed{}**************\".format(foldnum, s))\n        print(\"Gini:{:.6f}\".format(G))\n        print(\"Default:{:.6f}\".format(D))\n        print(\"Metric:{:.6f}\".format(M))\n        print()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:28.748273Z","iopub.execute_input":"2022-08-22T16:24:28.749213Z","iopub.status.idle":"2022-08-22T16:24:34.585790Z","shell.execute_reply.started":"2022-08-22T16:24:28.749175Z","shell.execute_reply":"2022-08-22T16:24:34.583867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def load_pickle_obj(filename):\n    with open(filename, 'rb') as file:\n        obj = pickle.load(file)\n    return obj\n\ntest_id2customer = load_pickle_obj(\"../input/amex-datasetcategorical-encoders/test_id2customer.pkl\")\nprint(len(test_id2customer))","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:34.587045Z","iopub.status.idle":"2022-08-22T16:24:34.587657Z","shell.execute_reply.started":"2022-08-22T16:24:34.587391Z","shell.execute_reply":"2022-08-22T16:24:34.587417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models=[]\nfor model_name in os.listdir('models'):\n    print(model_name)\n    model = torch.load(\"models/\"+model_name)\n    models.append(model)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:34.589710Z","iopub.status.idle":"2022-08-22T16:24:34.590229Z","shell.execute_reply.started":"2022-08-22T16:24:34.590007Z","shell.execute_reply":"2022-08-22T16:24:34.590029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = []\nfor fileid in range(60):\n    customer_path = \"../input/amex-sequence-test-dataset-v3/test_sequence/all_test_customerIds_{}.npy\".format(fileid)\n    xnum_path = \"../input/amex-sequence-test-dataset-v3/test_sequence/all_test_num_feats_{}.npy\".format(fileid)\n    xcat_path = \"../input/amex-sequence-test-dataset-v3/test_sequence/all_test_cat_feats_{}.npy\".format(fileid)\n    seqlen_path = \"../input/amex-sequence-test-dataset-v3/test_sequence/all_test_seqlens_{}.npy\".format(fileid)\n    \n    if os.path.exists(xnum_path):\n        customerids = np.load(customer_path)\n        xnum = np.load(xnum_path)\n        xcat = np.load(xcat_path)\n        xnum = np.clip(xnum, -6, 6)\n        xnum = np.nan_to_num(xnum, nan=-1)\n        \n        seqlens=np.load(seqlen_path)\n        \n        test_ids = np.arange(len(xnum))\n        test_dataset = AmexDataset(xnum, xcat, None, test_ids, allseqlens=seqlens, phase=\"infer\")\n        test_loader  = torch.utils.data.DataLoader(test_dataset, shuffle=False, drop_last=False, batch_size=2048)\n        \n        all_preds=[]\n        for xnum, xmissing, xcat, mask in test_loader:\n            xnum = xnum.to(device)\n            xmissing = xmissing.to(device)\n            xcat = xcat.to(device)\n            mask = mask.to(device)\n            \n            preds=np.zeros(len(xnum))\n            for model in models:\n                model.eval()\n                with torch.no_grad():\n                    outputs = model(xnum, xmissing, xcat, mask)\n                    yhat = outputs['y'].sigmoid()\n                    preds += yhat.cpu().numpy()\n            preds = preds/len(models)\n            all_preds += list(preds)\n        \n        df = pd.DataFrame.from_dict({\n            'customer_ID': customerids,\n            'prediction': all_preds\n        })\n        df.fillna(0.0, inplace=True)\n        sub_df.append(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:34.591831Z","iopub.status.idle":"2022-08-22T16:24:34.592338Z","shell.execute_reply.started":"2022-08-22T16:24:34.592114Z","shell.execute_reply":"2022-08-22T16:24:34.592138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=pd.concat(sub_df)\nsub_df['customer_ID'] = sub_df['customer_ID'].apply(lambda k: test_id2customer[k])\nsub_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:34.594072Z","iopub.status.idle":"2022-08-22T16:24:34.594541Z","shell.execute_reply.started":"2022-08-22T16:24:34.594299Z","shell.execute_reply":"2022-08-22T16:24:34.594319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-22T16:24:34.597096Z","iopub.status.idle":"2022-08-22T16:24:34.597737Z","shell.execute_reply.started":"2022-08-22T16:24:34.597410Z","shell.execute_reply":"2022-08-22T16:24:34.597449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T15:46:34.019988Z","iopub.status.idle":"2022-08-22T15:46:34.020549Z","shell.execute_reply.started":"2022-08-22T15:46:34.020266Z","shell.execute_reply":"2022-08-22T15:46:34.020294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}