{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, copy, gc\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import MultiStepLR\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.metrics import accuracy_score\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score, average_precision_score\nimport matplotlib.pyplot as plt\nimport random\nprint('Using PyTorch version',torch.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-24T08:01:58.560581Z","iopub.execute_input":"2022-08-24T08:01:58.561331Z","iopub.status.idle":"2022-08-24T08:02:01.082766Z","shell.execute_reply.started":"2022-08-24T08:01:58.561228Z","shell.execute_reply":"2022-08-24T08:02:01.081719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(seed=45)\n\ndevice = ('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.085044Z","iopub.execute_input":"2022-08-24T08:02:01.085899Z","iopub.status.idle":"2022-08-24T08:02:01.154574Z","shell.execute_reply.started":"2022-08-24T08:02:01.085849Z","shell.execute_reply":"2022-08-24T08:02:01.153390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AMEXLoader:\n\n    def __init__(self, X_3D, X_2D, y, lag, shuffle=True, batch_size=1024):\n        self.X_3D = X_3D\n        self.X_2D = X_2D\n        self.y = y\n\n        self.shuffle = shuffle\n        self.batch_size = batch_size\n        self.n_conts = self.X_3D.shape[1]\n        self.len = self.X_3D.shape[0]\n        n_batches, remainder = divmod(self.len, self.batch_size)\n\n        if remainder > 0:\n            n_batches += 1\n        self.n_batches = n_batches\n        self.remainder = remainder  # for debugging\n\n        self.idxes = np.array([i for i in range(self.len)])\n\n    def __iter__(self):\n        self.i = 0\n        if self.shuffle:\n            ridxes = self.idxes\n            np.random.shuffle(ridxes)\n            self.X_3D = self.X_3D[ridxes]\n            self.X_2D = self.X_2D[ridxes]\n            if self.y is not None:\n                self.y = self.y[ridxes]\n\n        return self\n\n    def __next__(self):\n        if self.i >= self.len:\n            raise StopIteration\n        \n        X_3D = torch.FloatTensor(self.X_3D[self.i:self.i + self.batch_size, lag:, :])\n        X_2D = torch.FloatTensor(self.X_2D[self.i:self.i + self.batch_size, :])\n        #idx = np.random.randint(self.len, size=xcont1.shape[0])\n        if self.y is not None:\n            y1 = self.y[self.i:self.i + self.batch_size]\n            #y2 = self.y[idx]\n            #y = torch.FloatTensor(np.where(y1==y2, 1, 0).astype(np.float32))\n            y1 = torch.FloatTensor(y1.astype(np.float32))\n\n        else:\n            y1 = None\n            #y = None\n            \n        #xcont2 = torch.FloatTensor(self.X_cont[idx, :, :])\n        \n\n        batch = (X_3D, X_2D, y1)#, xcont2, y)\n        self.i += self.batch_size\n        return batch\n\n    def __len__(self):\n        return self.n_batches","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.157363Z","iopub.execute_input":"2022-08-24T08:02:01.158101Z","iopub.status.idle":"2022-08-24T08:02:01.170173Z","shell.execute_reply.started":"2022-08-24T08:02:01.158064Z","shell.execute_reply":"2022-08-24T08:02:01.169134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_weights(m):\n    if type(m) == nn.Linear:\n        torch.nn.init.xavier_uniform(m.weight)\n        m.bias.data.fill_(0.01)\n\n####### LSTM model for 3D input ############################\nclass LSTM_AMEX(nn.Module):\n\n    def __init__(self, input_size, ffnn_input, hidden_size, \n                 num_layers, seq_length, activation = nn.GELU(), device=device):\n        \n        super(LSTM_AMEX, self).__init__()\n\n        self.num_layers = num_layers\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.seq_length = seq_length\n        self.device = device\n\n\n        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,\n                            num_layers=num_layers, batch_first=True)\n\n        \n        inp_dim = hidden_size\n        \n        self.ffnn = nn.Sequential(nn.Linear(ffnn_input, 512), \n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(512, 256),\n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(256, 128)\n                                        )\n        \n        \n        self.classifier = nn.Sequential(nn.Linear(inp_dim+128+input_size, 128), \n                                        nn.Dropout(0.20),\n                                        activation,\n                                        nn.Linear(128, 64),\n                                        #nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(64, 1), \n                                        nn.Sigmoid()\n                                        )\n\n        \n        self.attention1 = nn.Sequential(\n                            nn.Linear(inp_dim, 256),\n                            nn.Tanh(),\n                            nn.Linear(256, 1),\n                            nn.Softmax(dim=1)\n                        )\n        \n\n    def forward(self, inp3D, inp2D):#, cont_x2):\n        \n        inp3D = inp3D.to(self.device)\n        inp2D = inp2D.to(self.device)\n      \n        h_0 = Variable(torch.zeros(\n            self.num_layers, inp3D.size(0), self.hidden_size)).to(device)\n\n        c_0 = Variable(torch.zeros(\n            self.num_layers, inp3D.size(0), self.hidden_size)).to(device)\n\n\n        # Propagate input through LSTM\n        x, _ = self.lstm(inp3D, (h_0, c_0))\n        weights1 = self.attention1(x)\n        x = torch.sum(weights1 * x, dim=1)\n        \n        x2 = self.ffnn(inp2D)\n        x = torch.cat([x, x2, inp3D[:, -1, :]], dim=1)\n        \n        out = self.classifier(x)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.172216Z","iopub.execute_input":"2022-08-24T08:02:01.172786Z","iopub.status.idle":"2022-08-24T08:02:01.188111Z","shell.execute_reply.started":"2022-08-24T08:02:01.172748Z","shell.execute_reply":"2022-08-24T08:02:01.186800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### Custom Model ############################\n\n# Fully connected neural network with one hidden layer\nclass MLP_MODEL(nn.Module):\n    def __init__(self, input_size, hidden_size,\n                 num_layers, num_output, activation, attention=False):\n        super(MLP_MODEL, self).__init__()\n        self.meta_model = torch.nn.ModuleList()\n        for i in range(num_layers):\n            if i==0:\n                self.meta_model.append(nn.Linear(input_size, hidden_size))\n                #self.meta_model.append(nn.BatchNorm1d(hidden_size))\n                self.meta_model.append(nn.Dropout(0.10))\n                self.meta_model.append(activation)\n            else:\n                self.meta_model.append(nn.Linear(hidden_size, hidden_size))\n                #self.meta_model.append(nn.BatchNorm1d(hidden_size))\n                self.meta_model.append(nn.Dropout(0.10))\n                self.meta_model.append(activation)\n\n        self.linear = nn.Linear(hidden_size, num_output)\n\n        self.attention = attention\n        if self.attention == True:\n            self.att = nn.Sequential(\n                nn.Linear(input_size, hidden_size),\n                activation,\n                nn.Linear(hidden_size, input_size),\n                nn.Sigmoid()\n            )\n\n    def forward(self, x):\n\n        if self.attention==True:\n            w = self.att(x)\n            x = w*x\n        else:\n            x = x\n\n        global out\n        for i in range(len(self.meta_model)):\n            if i== 0:\n                out = self.meta_model[0](x)\n            else:\n                out = self.meta_model[i](out)\n\n        out = self.linear(out)\n\n        return out\n\nclass AMEX_Model(nn.Module):\n\n    def __init__(self, input_size, ffnn_input, hidden_size, \n                 num_layers, seq_length, activation, device=device):\n        \n        super(AMEX_Model, self).__init__()\n\n        self.device = device\n\n        self.encoders = torch.nn.ModuleList()\n        for i in range(seq_length):\n            self.encoders.append(MLP_MODEL(input_size=input_size,\n                                           hidden_size=hidden_size,\n                                           num_layers=num_layers,\n                                           num_output=64, \n                                           activation = activation))\n\n        self.ffnn = nn.Sequential(nn.Linear(ffnn_input, 256), \n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(256, 256),\n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(256, 256)\n                                        )\n        \n        \n        self.classifier = nn.Sequential(nn.Linear(64*seq_length+256+input_size, 256), \n                                        nn.Dropout(0.20),\n                                        activation,\n                                        nn.Linear(256, 128),\n                                        #nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(128, 1), \n                                        nn.Sigmoid()\n                                        )\n        \n        \n\n    def forward(self, inp3D, inp2D):\n        \n        inp3D = inp3D.to(self.device)\n        inp2D = inp2D.to(self.device)\n        \n        encoded_input = []\n        for i in range(inp3D.shape[1]):\n            encoded_input.append(self.encoders[i](inp3D[:, i, :]))\n        \n        encoded_input = torch.cat(encoded_input, dim=1)\n        \n        x = self.ffnn(inp2D)\n        x = torch.cat([encoded_input, x, inp3D[:, -1, :]], dim=1)\n        \n        out = self.classifier(x)\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.192037Z","iopub.execute_input":"2022-08-24T08:02:01.192296Z","iopub.status.idle":"2022-08-24T08:02:01.211053Z","shell.execute_reply.started":"2022-08-24T08:02:01.192272Z","shell.execute_reply":"2022-08-24T08:02:01.209956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 1D CNN model #### \nclass CNN_AMEX(nn.Module):\n    \n    def __init__(self, input_size, ffnn_input, hidden_size, num_layers, \n                 seq_length, activation, device=device):\n        super().__init__()\n\n        self.num_layers = num_layers\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.seq_length = seq_length\n        self.device = device\n        \n        def _norm(layer, dim=None):\n            return nn.utils.weight_norm(layer, dim=dim) \n\n        self.conv1 = nn.Sequential(\n            nn.BatchNorm1d(seq_length),\n            #nn.Dropout(0.10),\n            nn.Conv1d(seq_length, 8, kernel_size=1, stride=1, bias=True),\n            activation,\n            #nn.AdaptiveAvgPool1d(output_size=128),\n            nn.BatchNorm1d(8),\n            #nn.Dropout(0.10),\n            nn.Conv1d(8, 4, kernel_size=1, stride=1, bias=True),\n            activation,\n            #nn.AdaptiveAvgPool1d(output_size=64)\n        )\n\n        self.flt = nn.Flatten()\n        \n        self.ffnn = nn.Sequential(nn.Linear(ffnn_input, 128), \n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(128, 128),\n                                        nn.Dropout(0.10),\n                                        activation,\n                                        nn.Linear(128, 64)\n                                        )\n        \n\n        self.classifier = nn.Sequential(nn.Linear(5*input_size+64+ffnn_input, 256),\n                                        #nn.BatchNorm1d(512),\n                                        nn.Dropout(0.20),\n                                        activation,\n                                        nn.Linear(256, 128),\n                                        #nn.BatchNorm1d(256),\n                                        #nn.Dropout(0.20),\n                                        activation,\n                                        nn.Linear(128, 1), \n                                        nn.Sigmoid()\n                                        )\n \n\n    def forward(self, inp3D, inp2D):\n        \n        inp3D = inp3D.to(self.device)\n        inp2D = inp2D.to(self.device)\n\n        x1 = self.conv1(inp3D)\n        x1 = self.flt(x1)\n        \n        x2 = self.ffnn(inp2D)\n        x = torch.cat([x1, x2, inp3D[:, -1, :], inp2D], dim=1)\n        \n        out = self.classifier(x)\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.212621Z","iopub.execute_input":"2022-08-24T08:02:01.213134Z","iopub.status.idle":"2022-08-24T08:02:01.227800Z","shell.execute_reply.started":"2022-08-24T08:02:01.213098Z","shell.execute_reply":"2022-08-24T08:02:01.226695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TO_DATA='../input/amex-merging-and-fixing-test-data/'","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.231255Z","iopub.execute_input":"2022-08-24T08:02:01.231646Z","iopub.status.idle":"2022-08-24T08:02:01.240425Z","shell.execute_reply.started":"2022-08-24T08:02:01.231598Z","shell.execute_reply":"2022-08-24T08:02:01.239256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def inference(Nfolds, MODEL_TYPE, lag, seq_len, num_epoch=50, patience=10,\n                num_layers=3, activation=nn.GELU(), MODEL_ROOT='models/lstm/',\n                hidden_dim=512):\n    \n    y_pred = []\n    cids = []\n    for k in range(12):\n        \n        # READ TEST DATA FROM DISK\n        X_test_3D = []; X_test_2D = []; customers=[]\n        X_test_3D.append(np.load(f'{PATH_TO_DATA}train_num_{k}.npy'))\n        X_test_2D.append(np.concatenate((np.load(f'{PATH_TO_DATA}train_high_{k}.npy'), \n                                          np.load(f'{PATH_TO_DATA}train_skew_{k}.npy'), \n                                          #np.load(f'{PATH_TO_DATA}test_meduim_{k}.npy')\n                                         ), axis=1))\n        customers.append(pd.read_pickle(f'{PATH_TO_DATA}targets_{k}.pkl') )\n\n        X_test_3D = np.concatenate(X_test_3D,axis=0)\n        X_test_2D = np.concatenate(X_test_2D,axis=0)                                  \n        cids = cids + list(pd.concat(customers).customer_ID.values)\n        \n        test_loader = AMEXLoader(X_test_3D, X_test_2D, None, lag, batch_size=8196*4, shuffle=False)\n    \n        models = []\n        for fold in range(Nfolds):\n\n            model_path = MODEL_ROOT + f'/modeL_{fold}.pth'\n\n            if MODEL_TYPE=='CNN':\n                model = CNN_AMEX(input_size = X_test_3D.shape[2], ffnn_input = X_test_2D.shape[1], \n                                 hidden_size=hidden_dim, num_layers=num_layers, seq_length=seq_len, \n                                 activation=activation).to(device)\n\n                model = torch.load(model_path)\n                model.to(device)\n                models.append(model)\n\n            elif MODEL_TYPE=='LSTM':\n                model = LSTM_AMEX(input_size = X_test_3D.shape[2], ffnn_input = X_test_2D.shape[1], \n                                  hidden_size=hidden_dim, num_layers=num_layers, seq_length=seq_len, \n                                  activation=activation).to(device)\n                model = torch.load(model_path)\n                model.to(device)\n                models.append(model)\n\n            else:\n                model = AMEX_Model(input_size = X_test_3D.shape[2], ffnn_input = X_test_2D.shape[1],\n                                   hidden_size=256, num_layers=num_layers, seq_length=seq_len, \n                                   activation=activation).to(device)\n                model = torch.load(model_path)\n                model.to(device)\n                models.append(model)\n\n\n        \n        preds = np.zeros(X_test_2D.shape[0])\n        for model in models:\n            with torch.no_grad():\n                model.eval()\n                pred = []\n                for X_3D, X_2D, y in tqdm(test_loader):\n                    out = model(X_3D, X_2D)\n                    pred += list(out.detach().cpu().numpy().flatten())\n            preds += np.asarray(pred)/len(models)\n        \n        y_pred.append(preds)\n           \n    return cids, y_pred","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.241991Z","iopub.execute_input":"2022-08-24T08:02:01.242665Z","iopub.status.idle":"2022-08-24T08:02:01.258858Z","shell.execute_reply.started":"2022-08-24T08:02:01.242629Z","shell.execute_reply":"2022-08-24T08:02:01.257739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"preds = []\nlag = 0\nfor MODEL_TYPE in ['MLP', 'CNN', 'LSTM']:\n    cids, y_pred = inference(Nfolds=5, MODEL_TYPE=MODEL_TYPE, lag=lag, \n                              seq_len=13-lag, num_epoch=20, patience=5,\n                              num_layers=2, activation = nn.GELU(), \n                              MODEL_ROOT=f'../input/amex-neural-network-model/models/{MODEL_TYPE}/',\n                              hidden_dim=256)\n\n    \n    sub = pd.DataFrame()\n    sub['customer_ID'] = cids\n    sub['prediction'] = np.concatenate(y_pred)\n    preds.append(sub)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:02:01.262183Z","iopub.execute_input":"2022-08-24T08:02:01.262512Z","iopub.status.idle":"2022-08-24T08:05:51.919104Z","shell.execute_reply.started":"2022-08-24T08:02:01.262486Z","shell.execute_reply":"2022-08-24T08:05:51.918092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"import pickle\nwith open('../input/amex-merging-and-fixing-test-data/encoder.pkl', 'rb') as fin:\n    encoder = pickle.load(fin)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:05:51.920842Z","iopub.execute_input":"2022-08-24T08:05:51.921222Z","iopub.status.idle":"2022-08-24T08:05:53.014289Z","shell.execute_reply.started":"2022-08-24T08:05:51.921182Z","shell.execute_reply":"2022-08-24T08:05:53.013369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.DataFrame(index=preds[0]['customer_ID'].values, data={'prediction':0.30*preds[0]['prediction'].values + \n                                                                            0.35*preds[1]['prediction'].values + \n                                                                            0.35*preds[2]['prediction'].values})\n\nsub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\nsub['customer_ID_hash'] = encoder.transform(sub['customer_ID'])\nsub = sub.set_index('customer_ID_hash')\nsub = sub.merge(test[['prediction']], left_index=True, right_index=True, how='left')\nsub = sub.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:05:53.019040Z","iopub.execute_input":"2022-08-24T08:05:53.021016Z","iopub.status.idle":"2022-08-24T08:05:56.696997Z","shell.execute_reply.started":"2022-08-24T08:05:53.020972Z","shell.execute_reply":"2022-08-24T08:05:56.696012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DISPLAY PREDICTIONS\nsub.to_csv('submission_nn.csv',index=False)\nprint('Submission file shape is', sub.shape )\nprint(sub.isnull().sum())\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T08:05:56.698322Z","iopub.execute_input":"2022-08-24T08:05:56.698689Z","iopub.status.idle":"2022-08-24T08:05:59.520943Z","shell.execute_reply.started":"2022-08-24T08:05:56.698653Z","shell.execute_reply":"2022-08-24T08:05:59.519751Z"},"trusted":true},"execution_count":null,"outputs":[]}]}