{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport pickle\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\n\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T09:40:37.527961Z","iopub.execute_input":"2022-07-06T09:40:37.528403Z","iopub.status.idle":"2022-07-06T09:40:39.030899Z","shell.execute_reply.started":"2022-07-06T09:40:37.528297Z","shell.execute_reply":"2022-07-06T09:40:39.029544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Gru Baseline Reference:\nhttps://www.kaggle.com/code/cdeotte/tensorflow-gru-starter-0-790/notebook","metadata":{}},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    BATCH_SIZE=768\n    N_EPOCHS=15","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:40:39.524551Z","iopub.execute_input":"2022-07-06T09:40:39.525274Z","iopub.status.idle":"2022-07-06T09:40:39.531611Z","shell.execute_reply.started":"2022-07-06T09:40:39.525226Z","shell.execute_reply":"2022-07-06T09:40:39.530417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:40:41.210499Z","iopub.execute_input":"2022-07-06T09:40:41.210915Z","iopub.status.idle":"2022-07-06T09:40:41.217252Z","shell.execute_reply.started":"2022-07-06T09:40:41.210884Z","shell.execute_reply":"2022-07-06T09:40:41.216203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading the dataset","metadata":{}},{"cell_type":"code","source":"%%time\nallX=[]\nally=[]\n\nfor i in range(100):\n    xpath = \"../input/amex-train-series/Xchunk_{}.npy\".format(i)\n    ypath = \"../input/amex-train-series/ychunk_{}.npy\".format(i)\n    \n    if not os.path.exists(xpath):\n        break\n    allX.append(np.load(xpath))\n    ally.append(np.load(ypath))\n    gc.collect()\n    \nallX=np.concatenate(allX)\nally=np.concatenate(ally)\nprint(allX.shape, ally.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:40:43.377200Z","iopub.execute_input":"2022-07-06T09:40:43.377592Z","iopub.status.idle":"2022-07-06T09:40:51.697965Z","shell.execute_reply.started":"2022-07-06T09:40:43.377559Z","shell.execute_reply":"2022-07-06T09:40:51.696999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dataset","metadata":{}},{"cell_type":"code","source":"class AmexDataset(torch.utils.data.Dataset):\n    def __init__(self, X, y, idxs, phase='train'):\n        self.idxs=idxs\n        self.X = X\n        self.y = y\n        self.phase=phase\n    \n    def __getitem__(self, idx):\n        idx=self.idxs[idx]\n        Xnumeric = torch.tensor(self.X[idx][:, :-11], dtype=torch.float32)\n        Xcat  = torch.tensor(self.X[idx][:, -11:], dtype=torch.long)\n        if self.phase !='train':\n            return Xnumeric, Xcat\n        y = torch.tensor(self.y[idx], dtype=torch.float32)\n        return (Xnumeric, Xcat , y)\n    \n    def __len__(self):\n        return len(self.idxs)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:40:51.699475Z","iopub.execute_input":"2022-07-06T09:40:51.700097Z","iopub.status.idle":"2022-07-06T09:40:51.709936Z","shell.execute_reply.started":"2022-07-06T09:40:51.700062Z","shell.execute_reply":"2022-07-06T09:40:51.708930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self, sz):\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.BatchNorm1d(sz),\n            nn.Dropout(0.1),\n            nn.Linear(sz, sz//2),\n            nn.GELU(),\n            \n            nn.BatchNorm1d(sz//2),\n            nn.Dropout(0.1),\n            nn.Linear(sz//2, sz//4),\n            nn.GELU(),\n            nn.Linear(sz//4, 1)\n        )\n        \n    def forward(self, x):\n        return self.mlp(x)\n\nclass FeatureBlock(nn.Module):\n    def __init__(self, input_sz, out_sz):\n        super().__init__()\n        self.linear = nn.Linear(input_sz, out_sz)\n        self.bn = nn.BatchNorm1d(out_sz)\n        self.activation = nn.GELU()\n        self.dropout = nn.Dropout(0.1)\n            \n    def forward(self, x):\n        x = self.linear(x)\n        x = x.permute(0, 2, 1)\n        x = self.bn(x)\n        x = x.permute(0, 2, 1)\n        x = self.dropout(self.activation(x))\n        return x\n    \nclass FeatureTransform(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.dropout = nn.Dropout(0.1)\n        self.bn = nn.BatchNorm1d(213)\n        self.block1 = FeatureBlock(213, 128)\n        self.block2 = FeatureBlock(128, 128)\n        \n    def forward(self, x):\n        x = x.permute(0, 2, 1)\n        x = self.bn(x)\n        x = x.permute(0, 2, 1)\n        \n        x = self.block1(x)\n        x = self.block2(x)\n        return x\n\nclass AmexGruModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.embeddings = nn.ModuleList([nn.Embedding(10, 4, padding_idx=0) for _ in range(11)])\n        self.feature_transform = FeatureTransform()\n        self.gru = nn.GRU(128, 128, bidirectional = True, batch_first = True)\n        self.out1 = MLP(128)\n        self.out2 = MLP(128)\n        self.out  = MLP(2*128)\n        \n        \n    def forward(self, x, xcat):\n        xcat_embedds = []\n        for i in range(11):\n            xcat_embedds.append( self.embeddings[i](xcat[:, :, i]) )\n        xcat_embedds = torch.cat(xcat_embedds, dim=-1)\n        \n        x = torch.cat([x, xcat_embedds], dim=-1)\n        x = self.feature_transform(x)\n        (_, h) = self.gru(x)\n        h1 = h[0]\n        h2 = h[1]\n        \n        h = torch.cat([h1, h2], dim=-1)\n        \n        y1 = self.out1(h1).view(-1)\n        y2 = self.out2(h2).view(-1)\n        y = self.out(h).view(-1)\n        return y, y1, y2","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.206366Z","iopub.execute_input":"2022-07-06T09:51:50.206733Z","iopub.status.idle":"2022-07-06T09:51:50.231609Z","shell.execute_reply.started":"2022-07-06T09:51:50.206703Z","shell.execute_reply":"2022-07-06T09:51:50.230726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"def top_4percent(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    four_percent_cutoff = 0.04 * sum(df['weight'])\n    df['weight_cumsum'] = df['weight'].cumsum()\n    df_cutoff = df[df.weight_cumsum <= four_percent_cutoff]\n    \n    return df_cutoff['target'].sum()/df['target'].sum()\n\ndef weighted_gini(pred_df):\n    df = pred_df.copy()\n    df = df.sort_values('pred', ascending=False)\n    df['weight'] = df['target'].apply(lambda v: 20 if v==0 else 1)\n    df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n    total_pos = (df['target'] * df['weight']).sum()\n    df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n    df['lorentz'] = df['cum_pos_found'] / total_pos\n    df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n    return df['gini'].sum()\n\n\ndef normalized_gini(df):\n    df_true=df[['target']].copy()\n    df_true['pred'] = df_true['target'].copy()\n    \n    G = weighted_gini(df)/weighted_gini(df_true)\n    return G","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.321557Z","iopub.execute_input":"2022-07-06T09:51:50.322262Z","iopub.status.idle":"2022-07-06T09:51:50.334936Z","shell.execute_reply.started":"2022-07-06T09:51:50.322214Z","shell.execute_reply":"2022-07-06T09:51:50.333970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_smoothing(y, yhat):\n    y = torch.clamp(y, 0.01, 0.99)\n    loss = -y*torch.log(torch.sigmoid(yhat)) - (1-y) * torch.log(1-torch.sigmoid(yhat))\n    return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.435419Z","iopub.execute_input":"2022-07-06T09:51:50.436122Z","iopub.status.idle":"2022-07-06T09:51:50.443386Z","shell.execute_reply.started":"2022-07-06T09:51:50.436070Z","shell.execute_reply":"2022-07-06T09:51:50.442218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train model","metadata":{}},{"cell_type":"code","source":"def get_lr(epoch_num):\n    lrs = [1e-3, 1e-3, 1e-3, 1e-4, 1e-4, 1e-4, 1e-5, 1e-5]\n    if epoch_num < len(lrs):\n        return lrs[foldnum]\n    return 1e-5","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.563465Z","iopub.execute_input":"2022-07-06T09:51:50.564170Z","iopub.status.idle":"2022-07-06T09:51:50.570369Z","shell.execute_reply.started":"2022-07-06T09:51:50.564118Z","shell.execute_reply":"2022-07-06T09:51:50.569581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, val_dataloader):\n    model.eval()\n    ytrue=[]\n    ypred=[]\n    \n    for (Xnumeric, Xcat , y) in val_dataloader:\n        Xnumeric = Xnumeric.to(device)\n        Xcat = Xcat.to(device)\n        y = y.to(device)\n        \n        with torch.no_grad():\n            (yhat, yhat1, yhat2)=model(Xnumeric, Xcat)\n            yhat = yhat.sigmoid()\n            ytrue += y.cpu().tolist()\n            ypred += yhat.cpu().tolist()\n    \n    df = pd.DataFrame.from_dict({\n        'target': ytrue,\n        'pred': ypred\n    })\n    \n    G = normalized_gini(df[['target', 'pred']])\n    D = top_4percent(df[['target', 'pred']])\n    \n    M = (G+D)/2\n    return (G, D, M)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.684186Z","iopub.execute_input":"2022-07-06T09:51:50.684919Z","iopub.status.idle":"2022-07-06T09:51:50.694950Z","shell.execute_reply.started":"2022-07-06T09:51:50.684864Z","shell.execute_reply":"2022-07-06T09:51:50.694028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(foldnum, train_dataloader, val_dataloader):\n    best_eval=None\n    \n    model = AmexGruModel().to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=0.001)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, \n                                                           T_max = CFG.N_EPOCHS * len(train_dataloader), \n                                                           eta_min=1e-5)\n    \n    for e in range(CFG.N_EPOCHS):\n        epoch_loss=[]\n        model.train()\n        for it, (Xnumeric, Xcat , y) in enumerate(train_dataloader):\n            Xnumeric = Xnumeric.to(device)\n            Xcat = Xcat.to(device)\n            y = y.to(device)\n            \n            (yhat, yhat1, yhat2) = model(Xnumeric, Xcat)\n            loss1 = criterion(yhat, y)\n            loss2 = criterion(yhat1, y)\n            loss3 = criterion(yhat2, y)\n            loss = (loss1 + loss2 + loss3)/3\n            \n            optimizer.zero_grad(set_to_none=True)\n            loss.backward()\n            \n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1)\n            optimizer.step()\n            scheduler.step()\n\n            epoch_loss.append(loss.item())        \n        #Evaluating\n        (G, D, M) = evaluate(model, val_dataloader)\n        if best_eval is None or best_eval<M:\n            best_eval = M\n            torch.save(model, \"model{}.pt\".format(foldnum))\n        \n        \n        print(\"epoch:{} | loss:{:.4f}\".format(e, np.mean(epoch_loss)))\n        print(\"current Eval:{:.4f} | best Eval:{:.4f}\".format(M, best_eval))\n        print(\"Gini:{:.4f} | Default Rate:{:4f}\".format(G, D))\n        print()\n        print()\n        \n        plt.title(\"train epoch loss.\")\n        plt.plot(epoch_loss)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.810401Z","iopub.execute_input":"2022-07-06T09:51:50.811099Z","iopub.status.idle":"2022-07-06T09:51:50.825605Z","shell.execute_reply.started":"2022-07-06T09:51:50.811050Z","shell.execute_reply":"2022-07-06T09:51:50.824802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=33, shuffle=True)\nfor foldnum, (train_index, val_index) in enumerate(skf.split(ally, ally)):\n    if foldnum == 2:\n        break\n        \n    train_dataset = AmexDataset(allX,ally, train_index)\n    val_dataset = AmexDataset(allX, ally, val_index)\n    \n    train_dataloader = torch.utils.data.DataLoader(train_dataset, \n                                                   batch_size=CFG.BATCH_SIZE, \n                                                   shuffle=True,\n                                                   drop_last=True)\n    \n    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE, \n                                                   shuffle=False,\n                                                   drop_last=False)\n    \n    \n    print(\"Foldnumber:\", foldnum)\n    print(\"number of train iterations:\", len(train_dataloader))\n    print(\"number of val iterations:\", len(val_dataloader))\n    \n    train_model(foldnum, train_dataloader, val_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T09:51:50.926073Z","iopub.execute_input":"2022-07-06T09:51:50.926514Z","iopub.status.idle":"2022-07-06T09:51:55.721908Z","shell.execute_reply.started":"2022-07-06T09:51:50.926475Z","shell.execute_reply":"2022-07-06T09:51:55.720697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def load_pickle_obj(filename):\n    with open(filename, 'rb') as file:\n        obj = pickle.load(file)\n    return obj\n\ntest_id2customer = load_pickle_obj(\"../input/amex-datasetcategorical-encoders/test_id2customer.pkl\")\nprint(len(test_id2customer))","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:52:56.335283Z","iopub.execute_input":"2022-06-30T10:52:56.335621Z","iopub.status.idle":"2022-06-30T10:52:57.682718Z","shell.execute_reply.started":"2022-06-30T10:52:56.33559Z","shell.execute_reply":"2022-06-30T10:52:57.681546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models=[]\nfor i in range(2):\n    model = torch.load(\"model{}.pt\".format(i))\n    models.append(model)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:52:57.684375Z","iopub.execute_input":"2022-06-30T10:52:57.685018Z","iopub.status.idle":"2022-06-30T10:52:57.736338Z","shell.execute_reply.started":"2022-06-30T10:52:57.684972Z","shell.execute_reply":"2022-06-30T10:52:57.735369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(models)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:52:57.737769Z","iopub.execute_input":"2022-06-30T10:52:57.738115Z","iopub.status.idle":"2022-06-30T10:52:57.745967Z","shell.execute_reply.started":"2022-06-30T10:52:57.738081Z","shell.execute_reply":"2022-06-30T10:52:57.744461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = []\nfor fileid in range(200):\n    xpath = \"../input/amex-test-time-series-dataset/Xchunk_{}.npy\".format(fileid)\n    customer_path = \"../input/amex-test-time-series-dataset/customerIds_chunk_{}.npy\".format(fileid)\n    \n    if os.path.exists(xpath):\n        Xtest = np.load(xpath)\n        customerids = np.load(customer_path)\n        test_ids = np.arange(len(Xtest))\n        test_dataset = AmexDataset(Xtest, None, test_ids, phase=\"infer\")\n        test_loader  = torch.utils.data.DataLoader(test_dataset, shuffle=False, drop_last=False, batch_size=512)\n        \n        all_preds=[]\n        for (Xnumeric, Xcat) in test_loader:\n            Xnumeric = Xnumeric.to(device)\n            Xcat = Xcat.to(device)\n            preds=np.zeros(len(Xnumeric))\n            \n            for model in models:\n                model.eval()\n                with torch.no_grad():\n                    (yhat, _, _) = model(Xnumeric, Xcat)\n                    yhat = yhat.sigmoid()\n                    preds += yhat.cpu().numpy()\n            preds = preds/len(models)\n            all_preds += list(preds)\n        \n        df = pd.DataFrame.from_dict({\n            'customer_ID': customerids,\n            'prediction': all_preds\n        })\n        df.fillna(0.0, inplace=True)\n        sub_df.append(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:52:57.74769Z","iopub.execute_input":"2022-06-30T10:52:57.748183Z","iopub.status.idle":"2022-06-30T10:53:48.912244Z","shell.execute_reply.started":"2022-06-30T10:52:57.748139Z","shell.execute_reply":"2022-06-30T10:53:48.910604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=pd.concat(sub_df)\nsub_df['customer_ID'] = sub_df['customer_ID'].apply(lambda k: test_id2customer[k])\nsub_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:53:51.269684Z","iopub.execute_input":"2022-06-30T10:53:51.270072Z","iopub.status.idle":"2022-06-30T10:53:51.357707Z","shell.execute_reply.started":"2022-06-30T10:53:51.270041Z","shell.execute_reply":"2022-06-30T10:53:51.356556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:53:48.916533Z","iopub.status.idle":"2022-06-30T10:53:48.917377Z","shell.execute_reply.started":"2022-06-30T10:53:48.917074Z","shell.execute_reply":"2022-06-30T10:53:48.917103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T10:53:48.91926Z","iopub.status.idle":"2022-06-30T10:53:48.919847Z","shell.execute_reply.started":"2022-06-30T10:53:48.919554Z","shell.execute_reply":"2022-06-30T10:53:48.919581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(sub_df['prediction'], bins=100)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}