{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AMEX CosSquareFormer Starter\n\nBased on the transformer used in our [paper](https://www.ijcai.org/proceedings/2022/0698.pdf) recently published in IJCAI-22. (https://github.com/mayukh18/DEAP)\n\nAlso special thanks to Chris Deotte's wonderful notebooks, especially https://www.kaggle.com/code/cdeotte/tensorflow-gru-starter-0-790/notebook, which formed a portion of the skeleton of this notebook.","metadata":{}},{"cell_type":"code","source":"# all imports here\nimport os\nimport sys\nimport glob\nimport math\nimport random\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\n\nimport gc\nimport sklearn\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\nfrom sklearn import linear_model\nfrom sklearn import feature_selection\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom torch.autograd import Variable\n\nfrom datetime import datetime\nfrom tqdm import tqdm\nfrom copy import deepcopy\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndevice = 'cuda'\n\npd.set_option(\"display.max_columns\", None)\nplt.style.use(\"ggplot\")\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-25T00:52:10.562846Z","iopub.execute_input":"2022-06-25T00:52:10.56365Z","iopub.status.idle":"2022-06-25T00:52:13.733645Z","shell.execute_reply.started":"2022-06-25T00:52:10.56316Z","shell.execute_reply":"2022-06-25T00:52:13.732439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setting up the standard scaler","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\n\nx = []\nfor i in range(2):\n    x.append(np.load(f'../input/amex-data-for-transformers-and-rnns/data/data_{i+1}.npy'))\nx = np.concatenate(x, axis=0)\nscaler.fit(np.reshape(x, (-1, 188)))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:13.735979Z","iopub.execute_input":"2022-06-25T00:52:13.73726Z","iopub.status.idle":"2022-06-25T00:52:23.713958Z","shell.execute_reply.started":"2022-06-25T00:52:13.737186Z","shell.execute_reply":"2022-06-25T00:52:23.712963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Competition Metric","metadata":{}},{"cell_type":"code","source":"def amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.715683Z","iopub.execute_input":"2022-06-25T00:52:23.71624Z","iopub.status.idle":"2022-06-25T00:52:23.732257Z","shell.execute_reply.started":"2022-06-25T00:52:23.716193Z","shell.execute_reply":"2022-06-25T00:52:23.731192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformer","metadata":{}},{"cell_type":"code","source":"class MultiHeadAttention(nn.Module):\n    '''Multi-head self-attention module'''\n    def __init__(self, D, H):\n        super(MultiHeadAttention, self).__init__()\n        self.H = H # number of heads\n        self.D = D # dimension\n        \n        self.wq = nn.Linear(D, D*H)\n        self.wk = nn.Linear(D, D*H)\n        self.wv = nn.Linear(D, D*H)\n\n        self.dense = nn.Linear(D*H, D)\n\n    def concat_heads(self, x):\n        '''(B, H, S, D) => (B, S, D*H)'''\n        B, H, S, D = x.shape\n        x = x.permute((0, 2, 1, 3)).contiguous()  # (B, S, H, D)\n        x = x.reshape((B, S, H*D))   # (B, S, D*H)\n        return x\n\n    def split_heads(self, x):\n        '''(B, S, D*H) => (B, H, S, D)'''\n        B, S, D_H = x.shape\n        x = x.reshape(B, S, self.H, self.D)    # (B, S, H, D)\n        x = x.permute((0, 2, 1, 3))  # (B, H, S, D)\n        return x\n\n    def forward(self, x, mask):\n\n        q = self.wq(x)  # (B, S, D*H)\n        k = self.wk(x)  # (B, S, D*H)\n        v = self.wv(x)  # (B, S, D*H)\n\n        q = self.split_heads(q)  # (B, H, S, D)\n        k = self.split_heads(k)  # (B, H, S, D)\n        v = self.split_heads(v)  # (B, H, S, D)\n\n        attention_scores = torch.matmul(q, k.transpose(-1, -2)) #(B,H,S,S)\n        attention_scores = attention_scores / math.sqrt(self.D)\n\n        # add the mask to the scaled tensor.\n        if mask is not None:\n            attention_scores += (mask * -1e9)\n        \n        attention_weights = nn.Softmax(dim=-1)(attention_scores)\n        scaled_attention = torch.matmul(attention_weights, v)  # (B, H, S, D)\n        concat_attention = self.concat_heads(scaled_attention) # (B, S, D*H)\n        output = self.dense(concat_attention)  # (B, S, D)\n\n        return output, attention_weights\n\nclass MultiHeadAttention(nn.Module):\n    '''Multi-head self-attention module'''\n    def __init__(self, D, H):\n        super(MultiHeadAttention, self).__init__()\n        self.H = H # number of heads\n        self.D = D # dimension\n        \n        self.wq = nn.Linear(D, D*H)\n        self.wk = nn.Linear(D, D*H)\n        self.wv = nn.Linear(D, D*H)\n\n        self.dense = nn.Linear(D*H, D)\n\n    def concat_heads(self, x):\n        '''(B, H, S, D) => (B, S, D*H)'''\n        B, H, S, D = x.shape\n        x = x.permute((0, 2, 1, 3)).contiguous()  # (B, S, H, D)\n        x = x.reshape((B, S, H*D))   # (B, S, D*H)\n        return x\n\n    def split_heads(self, x):\n        '''(B, S, D*H) => (B, H, S, D)'''\n        B, S, D_H = x.shape\n        x = x.reshape(B, S, self.H, self.D)    # (B, S, H, D)\n        x = x.permute((0, 2, 1, 3))  # (B, H, S, D)\n        return x\n\n    def forward(self, x, mask):\n\n        q = self.wq(x)  # (B, S, D*H)\n        k = self.wk(x)  # (B, S, D*H)\n        v = self.wv(x)  # (B, S, D*H)\n\n        q = self.split_heads(q)  # (B, H, S, D)\n        k = self.split_heads(k)  # (B, H, S, D)\n        v = self.split_heads(v)  # (B, H, S, D)\n\n        attention_scores = torch.matmul(q, k.transpose(-1, -2)) #(B,H,S,S)\n        attention_scores = attention_scores / math.sqrt(self.D)\n\n        # add the mask to the scaled tensor.\n        if mask is not None:\n            attention_scores += (mask * -1e9)\n        \n        attention_weights = nn.Softmax(dim=-1)(attention_scores)\n        scaled_attention = torch.matmul(attention_weights, v)  # (B, H, S, D)\n        concat_attention = self.concat_heads(scaled_attention) # (B, S, D*H)\n        output = self.dense(concat_attention)  # (B, S, D)\n\n        return output, attention_weights\n\nclass MultiHeadAttentionCosformerNew(nn.Module):\n    '''Multi-head self-attention module'''\n    def __init__(self, D, H):\n        super(MultiHeadAttentionCosformerNew, self).__init__()\n        self.H = H # number of heads\n        self.D = D # dimension\n        \n        self.wq = nn.Linear(D, D*H)\n        self.wk = nn.Linear(D, D*H)\n        self.wv = nn.Linear(D, D*H)\n\n        self.dense = nn.Linear(D*H, D)\n\n    def concat_heads(self, x):\n        '''(B, H, S, D) => (B, S, D*H)'''\n        B, H, S, D = x.shape\n        x = x.permute((0, 2, 1, 3)).contiguous()  # (B, S, H, D)\n        x = x.reshape((B, S, H*D))   # (B, S, D*H)\n        return x\n\n    def split_heads(self, x):\n        '''(B, S, D*H) => (B, H, S, D)'''\n        B, S, D_H = x.shape\n        x = x.reshape(B, S, self.H, self.D)    # (B, S, H, D)\n        x = x.permute((0, 2, 1, 3))  # (B, H, S, D)\n        return x\n\n    def forward(self, x, mask):\n\n        q = self.wq(x)  # (B, S, D*H)\n        k = self.wk(x)  # (B, S, D*H)\n        v = self.wv(x)  # (B, S, D*H)\n\n        q = self.split_heads(q).permute(0,2,1,3)  # (B, S, H, D)\n        k = self.split_heads(k).permute(0,2,1,3)  # (B, S, H, D)\n        v = self.split_heads(v).permute(0,2,1,3)  # (B, S, H, D)\n        B = q.shape[0]\n        S = q.shape[1]\n\n        q = torch.nn.functional.elu(q) + 1 # Sigmoid torch.nn.ReLU()\n        k = torch.nn.functional.elu(k) + 1 # Sigmoid torch.nn.ReLU()\n\n        # q, k, v -> [batch_size, seq_len, n_heads, d_head]\n        cos = (torch.cos(1.57*torch.arange(S)/S).unsqueeze(0)).repeat(B,1).to(device)\n        sin = (torch.sin(1.57*torch.arange(S)/S).unsqueeze(0)).repeat(B,1).to(device)\n        # cos, sin -> [batch_size, seq_len]\n        q_cos = torch.einsum('bsnd,bs->bsnd', q, cos)\n        q_sin = torch.einsum('bsnd,bs->bsnd', q, sin)\n        k_cos = torch.einsum('bsnd,bs->bsnd', k, cos)\n        k_sin = torch.einsum('bsnd,bs->bsnd', k, sin)\n        # q_cos, q_sin, k_cos, k_sin -> [batch_size, seq_len, n_heads, d_head]\n\n        kv_cos = torch.einsum('bsnx,bsnz->bnxz', k_cos, v)\n        # kv_cos -> [batch_size, n_heads, d_head, d_head]\n        qkv_cos = torch.einsum('bsnx,bnxz->bsnz', q_cos, kv_cos)\n        # qkv_cos -> [batch_size, seq_len, n_heads, d_head]\n\n        kv_sin = torch.einsum('bsnx,bsnz->bnxz', k_sin, v)\n        # kv_sin -> [batch_size, n_heads, d_head, d_head]\n        qkv_sin = torch.einsum('bsnx,bnxz->bsnz', q_sin, kv_sin)\n        # qkv_sin -> [batch_size, seq_len, n_heads, d_head]\n\n        # denominator\n        denominator = 1.0 / (torch.einsum('bsnd,bnd->bsn', q_cos, k_cos.sum(axis=1))\n                             + torch.einsum('bsnd,bnd->bsn',\n                                            q_sin, k_sin.sum(axis=1))\n                             + 1e-5)\n        # denominator -> [batch_size, seq_len, n_heads]\n\n        O = torch.einsum('bsnz,bsn->bsnz', qkv_cos +\n                              qkv_sin, denominator).contiguous()\n        # output -> [batch_size, seq_len, n_heads, d_head]\n\n        concat_attention = self.concat_heads(O.permute(0,2,1,3)) # (B, S, D*H)\n        output = self.dense(concat_attention)  # (B, S, D)\n\n        return output, None\n\nclass MultiHeadAttentionCosSquareformerNew(nn.Module):\n    '''Multi-head self-attention module'''\n    def __init__(self, D, H):\n        super(MultiHeadAttentionCosSquareformerNew, self).__init__()\n        self.H = H # number of heads\n        self.D = D # dimension\n        \n        self.wq = nn.Linear(D, D*H)\n        self.wk = nn.Linear(D, D*H)\n        self.wv = nn.Linear(D, D*H)\n\n        self.dense = nn.Linear(D*H, D)\n\n    def concat_heads(self, x):\n        '''(B, H, S, D) => (B, S, D*H)'''\n        B, H, S, D = x.shape\n        x = x.permute((0, 2, 1, 3)).contiguous()  # (B, S, H, D)\n        x = x.reshape((B, S, H*D))   # (B, S, D*H)\n        return x\n\n    def split_heads(self, x):\n        '''(B, S, D*H) => (B, H, S, D)'''\n        B, S, D_H = x.shape\n        x = x.reshape(B, S, self.H, self.D)    # (B, S, H, D)\n        x = x.permute((0, 2, 1, 3))  # (B, H, S, D)\n        return x\n\n    def forward(self, x, mask):\n\n        q = self.wq(x)  # (B, S, D*H)\n        k = self.wk(x)  # (B, S, D*H)\n        v = self.wv(x)  # (B, S, D*H)\n\n        q = self.split_heads(q).permute(0,2,1,3)  # (B, S, H, D)\n        k = self.split_heads(k).permute(0,2,1,3)  # (B, S, H, D)\n        v = self.split_heads(v).permute(0,2,1,3)  # (B, S, H, D)\n        B = q.shape[0]\n        S = q.shape[1]\n\n        q = torch.nn.functional.elu(q) + 1 # Sigmoid torch.nn.ReLU()\n        k = torch.nn.functional.elu(k) + 1 # Sigmoid torch.nn.ReLU()\n\n        # q, k, v -> [batch_size, seq_len, n_heads, d_head]\n        cos = (torch.cos(3.1415*torch.arange(S)/S).unsqueeze(0)).repeat(B,1).to(device)\n        sin = (torch.sin(3.1415*torch.arange(S)/S).unsqueeze(0)).repeat(B,1).to(device)\n        # cos, sin -> [batch_size, seq_len]\n        q_cos = torch.einsum('bsnd,bs->bsnd', q, cos)\n        q_sin = torch.einsum('bsnd,bs->bsnd', q, sin)\n        k_cos = torch.einsum('bsnd,bs->bsnd', k, cos)\n        k_sin = torch.einsum('bsnd,bs->bsnd', k, sin)\n        # q_cos, q_sin, k_cos, k_sin -> [batch_size, seq_len, n_heads, d_head]\n\n        kv_cos = torch.einsum('bsnx,bsnz->bnxz', k_cos, v)\n        # kv_cos -> [batch_size, n_heads, d_head, d_head]\n        qkv_cos = torch.einsum('bsnx,bnxz->bsnz', q_cos, kv_cos)\n        # qkv_cos -> [batch_size, seq_len, n_heads, d_head]\n\n        kv_sin = torch.einsum('bsnx,bsnz->bnxz', k_sin, v)\n        # kv_sin -> [batch_size, n_heads, d_head, d_head]\n        qkv_sin = torch.einsum('bsnx,bnxz->bsnz', q_sin, kv_sin)\n        # qkv_sin -> [batch_size, seq_len, n_heads, d_head]\n\n        kv = torch.einsum('bsnx,bsnz->bnxz', k, v)\n        # kv -> [batch_size, n_heads, d_head, d_head]\n        qkv = torch.einsum('bsnx,bnxz->bsnz', q, kv)\n        # qkv_cos -> [batch_size, seq_len, n_heads, d_head]\n\n        # denominator\n        denominator = 1.0 / (torch.einsum('bsnd,bnd->bsn', q, k.sum(axis=1)) + torch.einsum('bsnd,bnd->bsn', q_cos, k_cos.sum(axis=1))\n                             + torch.einsum('bsnd,bnd->bsn',\n                                            q_sin, k_sin.sum(axis=1))\n                             + 1e-5)\n        # denominator -> [batch_size, seq_len, n_heads]\n\n        O = torch.einsum('bsnz,bsn->bsnz', qkv + qkv_cos +\n                              qkv_sin, denominator).contiguous()\n        # output -> [batch_size, seq_len, n_heads, d_head]\n\n        concat_attention = self.concat_heads(O.permute(0,2,1,3)) # (B, S, D*H)\n        output = self.dense(concat_attention)  # (B, S, D)\n\n        return output, None\n\n# Positional encodings\ndef get_angles(pos, i, D):\n    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(D))\n    return pos * angle_rates\n\n\ndef positional_encoding(D, position=20, dim=3, device=device):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                            np.arange(D)[np.newaxis, :],\n                            D)\n    # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n    # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n    if dim == 3:\n        pos_encoding = angle_rads[np.newaxis, ...]\n    elif dim == 4:\n        pos_encoding = angle_rads[np.newaxis,np.newaxis,  ...]\n    return torch.tensor(pos_encoding, device=device)\n\nclass TransformerLayer(nn.Module):\n    def __init__(self, D, H, hidden_mlp_dim, dropout_rate, attention_type='cosine_square'):\n        super(TransformerLayer, self).__init__()\n        self.dropout_rate = dropout_rate\n        self.mlp_hidden = nn.Linear(D, hidden_mlp_dim)\n        self.mlp_out = nn.Linear(hidden_mlp_dim, D)\n        self.layernorm1 = nn.LayerNorm(D, eps=1e-9)\n        self.layernorm2 = nn.LayerNorm(D, eps=1e-9)\n        self.dropout1 = nn.Dropout(dropout_rate)\n        self.dropout2 = nn.Dropout(dropout_rate)\n\n        if attention_type == 'cosine':\n          self.mha = MultiHeadAttentionCosformerNew(D, H)\n        elif attention_type == 'cosine_square':\n          self.mha = MultiHeadAttentionCosSquareformerNew(D, H)\n        else:\n          self.mha = MultiHeadAttention(D,H)\n\n    def forward(self, x, look_ahead_mask):\n        \n        attn, attn_weights = self.mha(x, look_ahead_mask)  # (B, S, D)\n        attn = self.dropout1(attn) # (B,S,D)\n        attn = self.layernorm1(attn + x) # (B,S,D)\n\n        mlp_act = torch.relu(self.mlp_hidden(attn))\n        mlp_act = self.mlp_out(mlp_act)\n        mlp_act = self.dropout2(mlp_act)\n        \n        output = self.layernorm2(mlp_act + attn)  # (B, S, D)\n\n        return output, attn_weights\n  \nclass Transformer(nn.Module):\n    '''Transformer Decoder Implementating several Decoder Layers.\n    '''\n    def __init__(self, num_layers, D, H, hidden_mlp_dim, inp_features, out_features, dropout_rate, attention_type='cosine_square'):\n        super(Transformer, self).__init__()\n        self.attention_type = attention_type\n        self.sqrt_D = torch.tensor(math.sqrt(D))\n        self.num_layers = num_layers\n        self.input_projection = nn.Linear(inp_features, D) # multivariate input\n        self.output_projection = nn.Linear(D, 96) # multivariate output\n        self.output_projection1 = nn.Linear(96, 16)\n        self.output_projection2 = nn.Linear(16, out_features)\n        self.pos_encoding = positional_encoding(D)\n        self.dec_layers = nn.ModuleList([TransformerLayer(D, H, hidden_mlp_dim, \n                                        dropout_rate=dropout_rate, attention_type=self.attention_type\n                                       ) for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout_rate)\n        self.sig = torch.nn.Sigmoid()\n\n    def forward(self, x, mask):\n        B, S, D = x.shape\n        # attention_weights = {}\n        x = self.input_projection(x)\n        x *= self.sqrt_D\n        \n        x += self.pos_encoding[:, :S, :]\n\n        x = self.dropout(x)\n\n        for i in range(self.num_layers):\n            x, _ = self.dec_layers[i](x=x,\n                                          look_ahead_mask=mask)\n            # attention_weights['decoder_layer{}'.format(i + 1)] = block\n        \n        x = self.output_projection(x)\n        x = self.output_projection1(x)\n        x = self.output_projection2(x)\n        \n        #return self.sig(x), None # attention_weights # (B,S,S)\n        return self.sig(x)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.734201Z","iopub.execute_input":"2022-06-25T00:52:23.734689Z","iopub.status.idle":"2022-06-25T00:52:23.809239Z","shell.execute_reply.started":"2022-06-25T00:52:23.734642Z","shell.execute_reply":"2022-06-25T00:52:23.80845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function that implement the look_ahead mask for masking future time steps. \ndef create_look_ahead_mask(size, device=device):\n    mask = torch.ones((size, size), device=device)\n    mask = torch.triu(mask, diagonal=1)\n    return mask  # (size, size)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.811083Z","iopub.execute_input":"2022-06-25T00:52:23.811457Z","iopub.status.idle":"2022-06-25T00:52:23.816998Z","shell.execute_reply.started":"2022-06-25T00:52:23.81142Z","shell.execute_reply":"2022-06-25T00:52:23.816292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- rfe_selector.get_support()\nrfe_selector.get_support()\narray([ True, False,  True,  True,  True,  True,  True,  True, False,\n        True, False,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True, False,  True,  True,  True,  True,  True,\n       False,  True,  True,  True,  True,  True, False,  True,  True,\n        True,  True, False,  True,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True, False,\n       False,  True,  True,  True,  True,  True, False,  True,  True,\n       False,  True,  True,  True, False,  True,  True, False,  True,\n        True,  True,  True, False,  True,  True,  True,  True, False,\n        True,  True,  True,  True,  True,  True, False,  True,  True,\n        True,  True,  True,  True,  True,  True,  True,  True, False,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True,  True, False,  True,  True,  True,  True,  True,\n       False,  True,  True, False,  True,  True,  True, False,  True,\n        True,  True,  True,  True,  True,  True,  True,  True,  True,\n        True,  True, False,  True, False,  True,  True,  True,  True,\n        True, False,  True, False,  True,  True,  True,  True,  True,\n        True,  True,  True,  True,  True,  True, False, False,  True,\n        True,  True,  True, False,  True,  True,  True,  True]) -->","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ntrain_ids = []\nval_ids = []\n\nkf = KFold(shuffle=True, random_state=42)\nfor x in kf.split(list(range(1,11))):\n    train_ids.append(x[0] + 1)\n    val_ids.append(x[1] + 1)\n    \ntrain_ids, val_ids","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.819131Z","iopub.execute_input":"2022-06-25T00:52:23.819907Z","iopub.status.idle":"2022-06-25T00:52:23.834317Z","shell.execute_reply.started":"2022-06-25T00:52:23.819872Z","shell.execute_reply":"2022-06-25T00:52:23.833665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"#cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\nDATA_PATH = '../input/amex-data-for-transformers-and-rnns'\n\nlr = 0.0001\nn_epochs = 16\nBATCH_SIZE = 128\nN_FEATS = 188\n\n\nsys.stderr.flush()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.837403Z","iopub.execute_input":"2022-06-25T00:52:23.838171Z","iopub.status.idle":"2022-06-25T00:52:23.953968Z","shell.execute_reply.started":"2022-06-25T00:52:23.838133Z","shell.execute_reply":"2022-06-25T00:52:23.953228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Start model training')\n\ncriterion = nn.BCELoss(reduction='none')\nFOLDS = list(range(5))\n\nfor fold in FOLDS:\n    print(f'============================= | Fold: {fold} |=============================')\n    \n    model = Transformer(num_layers=1, D=96, H=8, hidden_mlp_dim=3072, inp_features=3*N_FEATS, out_features=1, dropout_rate=0.25, attention_type='cosine_square').to(device)\n    opt = torch.optim.Adam(model.parameters(), lr=lr)\n    sched = torch.optim.lr_scheduler.StepLR(opt, step_size=2, gamma=0.8)\n\n    best_score = 0.0  \n    for epoch in range(n_epochs):\n        epoch_loss = 0\n        batch_idx = 0\n        model.train()\n\n        bar = tqdm(train_ids[fold])\n        for i in bar:\n            x_chunk = np.load(f'{DATA_PATH}/data/data_{i}.npy')\n            x_chunk = scaler.transform(np.reshape(x_chunk, (-1, N_FEATS)))\n            x_chunk = np.reshape(x_chunk, (-1, 13, N_FEATS))\n\n            x_chunk_mean = np.mean(x_chunk, axis=0, keepdims=True)\n            x_chunk_mean = np.repeat(x_chunk_mean, repeats=x_chunk.shape[0], axis=0)\n            \n            x_chunk = np.concatenate([x_chunk, x_chunk_mean, x_chunk - x_chunk_mean], axis=-1)\n            y_chunk = np.array(pd.read_parquet(f'{DATA_PATH}/data/targets_{i}.pqt')['target'])\n\n            batches = x_chunk.shape[0]//BATCH_SIZE\n            for b in range(batches):\n                start = b*BATCH_SIZE\n\n                opt.zero_grad()\n                x = torch.tensor(x_chunk[start: start+BATCH_SIZE, :]).float().to(device)\n                y = torch.tensor(y_chunk[start: start+BATCH_SIZE]).float().to(device)\n\n                mask = create_look_ahead_mask(x.shape[1]).to(device)\n                out = model(x, mask)\n\n                loss = criterion(out[:,-1,:], y.unsqueeze(1))\n                loss = torch.mean(loss)\n\n                epoch_loss = (epoch_loss*batch_idx + loss.item())/(batch_idx+1)\n                loss.backward()\n                opt.step()\n\n                bar.set_description(f'Train Id: {i}, Loss: {str(epoch_loss)}')\n                batch_idx += 1\n\n        sched.step()\n        model.eval()\n        all_true = []\n        all_pred = []\n\n        print(f'>> Epoch {epoch}: Validation')\n        for i in val_ids[fold]:\n            x_chunk = np.load(f'{DATA_PATH}/data/data_{i}.npy')\n            x_chunk = scaler.transform(np.reshape(x_chunk, (-1, N_FEATS)))\n            x_chunk = np.reshape(x_chunk, (-1, 13, N_FEATS))\n\n            x_chunk_mean = np.mean(x_chunk, axis=0, keepdims=True)\n            x_chunk_mean = np.repeat(x_chunk_mean, repeats=x_chunk.shape[0], axis=0)\n            \n            x_chunk = np.concatenate([x_chunk, x_chunk_mean, x_chunk - x_chunk_mean], axis=-1)\n            y_chunk = np.array(pd.read_parquet(f'{DATA_PATH}/data/targets_{i}.pqt')['target'])\n\n            batches = x_chunk.shape[0]//BATCH_SIZE\n            \n            with torch.no_grad():\n                for b in range(batches):\n                    start = b*BATCH_SIZE\n\n                    opt.zero_grad()\n                    x = torch.tensor(x_chunk[start: start+BATCH_SIZE, :]).float().to(device)\n                    y = torch.tensor(y_chunk[start: start+BATCH_SIZE]).float().to(device)\n\n                    mask = create_look_ahead_mask(x.shape[1]).to(device)\n                    out = model(x, mask)\n\n                    ytrue = y.squeeze().cpu().numpy()\n                    ypred = out[:,-1,0].squeeze().cpu().numpy()\n                    all_true.extend(ytrue)\n                    all_pred.extend(ypred)\n\n        all_true = np.array(all_true)\n        all_pred = np.array(all_pred)\n        \n        epoch_score = amex_metric_numpy(all_true, all_pred)\n        print('>> Eval Amex: {}: '.format(epoch_score))\n\n        if epoch_score > best_score:\n          best_score = epoch_score\n          print(\">> Saving model ...\")\n          torch.save(model.state_dict(), f'model_cossqformer_fold_{fold}')\n        \n        \n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-25T00:52:23.956428Z","iopub.execute_input":"2022-06-25T00:52:23.957081Z","iopub.status.idle":"2022-06-25T02:10:52.133845Z","shell.execute_reply.started":"2022-06-25T00:52:23.957043Z","shell.execute_reply":"2022-06-25T02:10:52.132946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ndel all_true, all_pred\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T02:10:52.135027Z","iopub.execute_input":"2022-06-25T02:10:52.135902Z","iopub.status.idle":"2022-06-25T02:10:52.261047Z","shell.execute_reply.started":"2022-06-25T02:10:52.135863Z","shell.execute_reply":"2022-06-25T02:10:52.260022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(f'../input/amex-default-prediction/sample_submission.csv')\nsub['hash'] = sub['customer_ID'].str[-16:].apply(lambda x: int(x, 16)).astype('int64')\ntest_hash_index = np.load(f'{DATA_PATH}/data/test_hashes_data.npy')\nsub = sub.set_index('hash').loc[test_hash_index].reset_index(drop=True)\n\nall_preds = np.zeros(len(sub))\n\nfor fold in FOLDS:\n    model = Transformer(num_layers=1, D=96, H=8, hidden_mlp_dim=3072, inp_features=3*N_FEATS, out_features=1, dropout_rate=0.2, attention_type='cosine_square').to(device)\n    model.load_state_dict(torch.load(f'model_cossqformer_fold_{fold}'))\n    model.eval()\n    \n    fold_preds = []\n    \n    print(f'Fold ID {fold}')\n    \n    bar = tqdm(range(1,21))\n    for i in bar:\n        x_chunk = np.load(f'{DATA_PATH}/data/test_data_{i}.npy')\n        x_chunk = scaler.transform(np.reshape(x_chunk, (-1, N_FEATS)))\n        x_chunk = np.reshape(x_chunk, (-1, 13, N_FEATS))\n\n        x_chunk_mean = np.mean(x_chunk, axis=0, keepdims=True)\n        x_chunk_mean = np.repeat(x_chunk_mean, repeats=x_chunk.shape[0], axis=0)\n        \n        x_chunk = np.concatenate([x_chunk, x_chunk_mean, x_chunk - x_chunk_mean], axis=-1)\n\n        batches = x_chunk.shape[0]//BATCH_SIZE + int((x_chunk.shape[0] % BATCH_SIZE) > 0)\n        chunk_preds = []\n        \n        with torch.no_grad():\n            for b in range(batches):\n                start = b*BATCH_SIZE\n                x = torch.tensor(x_chunk[start: min(start+BATCH_SIZE, len(x_chunk)), :]).float().to(device)\\\n\n                mask = create_look_ahead_mask(x.shape[1]).to(device)\n                out = model(x, mask)\n                ypred = out[:,-1,0].squeeze().cpu().numpy()\n                chunk_preds.extend(ypred)\n\n        fold_preds.extend(chunk_preds)\n    \n    all_preds += np.array(fold_preds)\n        \nall_preds = all_preds / 5\nsub['prediction'] = all_preds","metadata":{"execution":{"iopub.status.busy":"2022-06-24T18:30:49.656057Z","iopub.execute_input":"2022-06-24T18:30:49.656549Z","iopub.status.idle":"2022-06-24T18:31:20.055037Z","shell.execute_reply.started":"2022-06-24T18:30:49.656507Z","shell.execute_reply":"2022-06-24T18:31:20.053813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-11T17:15:50.002007Z","iopub.execute_input":"2022-06-11T17:15:50.002379Z","iopub.status.idle":"2022-06-11T17:15:50.498442Z","shell.execute_reply.started":"2022-06-11T17:15:50.002347Z","shell.execute_reply":"2022-06-11T17:15:50.497484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-06-11T17:15:50.577677Z","iopub.execute_input":"2022-06-11T17:15:50.578253Z","iopub.status.idle":"2022-06-11T17:15:50.589018Z","shell.execute_reply.started":"2022-06-11T17:15:50.578219Z","shell.execute_reply":"2022-06-11T17:15:50.588064Z"},"trusted":true},"execution_count":null,"outputs":[]}]}