{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# https://www.kaggle.com/c/riiid-test-answer-prediction/discussion/210276\n# https://medium.com/inside-machine-learning/what-is-a-transformer-d07dd1fbec04\n\n# Multihead vs Transformer?\n# This notebook seems to indicate a Transformer consists of the encoder and decoder blocks:\n# https://www.kaggle.com/m10515009/saint-is-all-you-need-training-private-0-801\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport pandas as pd\nimport numpy as np\nimport sklearn.metrics\nimport tqdm\n\nimport matplotlib.pyplot\n\nimport torch\n\nsettings = {}\nsettings['seq_len'] = 160\nsettings['n_content_id'] = 13525\nsettings['batch_size'] = 100\nsettings['embed_dim'] = 200\nsettings['n_train_rows'] = 5 * 1000000\nsettings['device'] = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndtype = {'timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv'\n                       ,usecols=[1, 2, 3, 4, 7]\n                       ,dtype=dtype\n                       ,nrows = settings['n_train_rows']\n                      )\n\n# Keep only questions\ntrain_df = train_df[train_df.content_type_id == False]\n\n# Arrange by timestamp\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\n\n\n# Group each user\ntrain_group = train_df[['user_id', 'content_id', 'answered_correctly']]\\\n            .groupby('user_id')\\\n            .apply(lambda r: {'content_id' : r['content_id'].values\n                             ,'answered_correctly' : r['answered_correctly'].values\n                            })\n\ndel train_df\ngc.collect()\n\n\n# Make validation set\nval_idx = np.random.choice(train_group.index, int(.1 * train_group.shape[0]), replace=False)\nvalid_group = train_group[val_idx].copy()\ntrain_group.drop(valid_group.index, inplace=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class riiid_dataset(torch.utils.data.Dataset):\n    \n    def __init__(self, group, settings):\n        super(riiid_dataset, self).__init__()\n        self.seq_len = settings['seq_len']\n        self.group = group\n        \n        # Take out people with only 1 interaction\n        for user_id in self.group.index:\n            if len(self.group[user_id]['content_id']) < 2:\n                del self.group[user_id]\n        \n    def __len__(self):\n        return(len(self.group))\n    \n    def __getitem__(self, index):\n        # Get the relevant user row\n        sample = self.group.iloc[index]\n        \n        # Get contents as np.int64s\n        content_id = sample['content_id'].astype(np.int64)\n        answered_correctly = sample['answered_correctly'].astype(np.int64)\n        \n        # Helper function to pad vector\n        def pad(np_array, out_size=self.seq_len):\n            n_pad = out_size - len(np_array)\n            if n_pad > 0:\n                np_array = np.concatenate((np.full(n_pad, 0).astype(np.int64), np_array))\n            else:\n                np_array = np_array[:out_size]\n            return(np_array)\n                \n        content_id = pad(content_id)\n        answered_correctly = pad(answered_correctly)\n        prev_ac = pad(answered_correctly, self.seq_len + 1)\n        prev_ac = prev_ac[:-1]\n        \n        # Return\n        return({\n            'content_id' : content_id\n            ,'answered_correctly' : answered_correctly\n            ,'prev_ac' : prev_ac\n        })\n    \n\ntrain_dataset = riiid_dataset(group = train_group\n                              ,settings = settings\n                              )\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset\n                                                ,batch_size = settings['batch_size']\n                                                ,drop_last = True\n                                                ,shuffle = True\n                                                ,num_workers = 4\n                                               )\n\nvalid_dataset = riiid_dataset(group = valid_group\n                             ,settings = settings\n                             )\nvalid_dataloader = torch.utils.data.DataLoader(valid_dataset\n                                               ,batch_size = settings['batch_size']\n                                               ,drop_last = True\n                                              )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class encoder(torch.nn.Module):\n    def __init__(self, settings):\n        super(encoder, self).__init__()\n        self.embed_dim = settings['embed_dim']\n        self.n_content_id = settings['n_content_id']\n        self.seq_len = settings['seq_len']\n        self.device = settings['device']\n        \n        self.cid_embedding = torch.nn.Embedding(self.n_content_id, self.embed_dim)\n        self.pos_embedding = torch.nn.Embedding(self.seq_len, self.embed_dim)\n        self.multi_att = torch.nn.MultiheadAttention(embed_dim = self.embed_dim\n                                                     ,num_heads = 8\n                                                     ,dropout = 0.2)\n\n        self.lin_1 = torch.nn.Linear(self.embed_dim, self.embed_dim)\n        self.relu = torch.nn.ReLU()\n        self.lin_2 = torch.nn.Linear(self.embed_dim, self.embed_dim)\n        self.dropout = torch.nn.Dropout(0.2)\n            \n    def forward(self, batch):\n        # Content embedding\n        x = self.cid_embedding(batch['content_id'])\n        \n        # Position embedding\n        pos_id = torch.arange(x.shape[1])[None, :].to(self.device)\n        pos_x = self.pos_embedding(pos_id)\n        \n        # Add embeddings and permute\n        x = x + pos_x\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        \n        # MultiHead Attention and permute back\n        attn_mask = torch.from_numpy(np.triu(np.ones((self.seq_len,self.seq_len)), k=1)\\\n                                         .astype('bool')).to(self.device) # torch.triu does not have k argument\n        attn_output, _ = self.multi_att(x, x, x, attn_mask = attn_mask)\n        x = x + attn_output\n        \n        # Feed forward\n        x = self.lin_1(x)\n        x = self.relu(x)\n        x = self.lin_2(x)\n        x = self.dropout(x)\n\n        # Return\n        return(x)\n        \nclass decoder(torch.nn.Module):\n    def __init__(self, settings):\n        super(decoder, self).__init__()\n        self.embed_dim = settings['embed_dim']\n        self.seq_len = settings['seq_len']\n        self.device = settings['device']\n        \n        self.prev_ac_embedding = torch.nn.Embedding(10, self.embed_dim)\n        self.pos_embedding = torch.nn.Embedding(self.seq_len, self.embed_dim)\n        self.multi_att_1 = torch.nn.MultiheadAttention(embed_dim = self.embed_dim\n                                                     ,num_heads = 8\n                                                     ,dropout = 0.2)\n        self.multi_att_2 = torch.nn.MultiheadAttention(embed_dim = self.embed_dim\n                                                      ,num_heads = 8\n                                                      ,dropout = 0.2)\n        \n        self.lin_1 = torch.nn.Linear(self.embed_dim, self.embed_dim)\n        self.relu = torch.nn.ReLU()\n        self.lin_2 = torch.nn.Linear(self.embed_dim, self.embed_dim)\n        self.dropout = torch.nn.Dropout(0.2)        \n        \n    def forward(self, batch, x):\n        # Previous answered_correctly embedding\n        y = self.prev_ac_embedding(batch['prev_ac'])\n        \n        # Position embedding\n        pos_id = torch.arange(y.shape[1])[None, :].to(self.device)\n        pos_y = self.pos_embedding(pos_id)\n        \n        # Add embeddings and permute\n        y = y + pos_y\n        y = y.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        \n        # MultiHead Attention 1\n        attn_mask = torch.from_numpy(np.triu(np.ones((self.seq_len,self.seq_len)), k=1)\\\n                                         .astype('bool')).to(self.device) # torch.triu does not have k argument\n        \n        attn_output_1, _ = self.multi_att_1(y, y, y, attn_mask = attn_mask)\n        y = y + attn_output_1\n        \n        # MultiHead Attention 2\n        attn_output_2, _ = self.multi_att_2(y, x, x, attn_mask = attn_mask) # query, key, value\n        y = y + attn_output_2\n        \n        # Permute back to [batch_size, seq_len, embed_dim]\n        y = y.permute(1, 0, 2)\n        \n        # Feed forward\n        y = self.lin_1(y)\n        y = self.relu(y)\n        y = self.lin_2(y)\n        y = self.dropout(y)\n\n        # Return\n        return(y)\n        \nclass riiid_model(torch.nn.Module):\n    def __init__(self, settings):\n        super(riiid_model, self).__init__()\n        self.embed_dim = settings['embed_dim']\n        self.seq_len = settings['seq_len']\n        self.device = settings['seq_len']\n        self.encoder = encoder(settings=settings)\n        self.decoder = decoder(settings=settings)\n        self.emb_to_seq = torch.nn.Linear(self.embed_dim, 1)\n    \n    def forward(self, batch):\n        x = self.encoder(batch)\n        y = self.decoder(batch, x)\n        y = self.emb_to_seq(y)\n        y = y[:,:,0]\n        return(y)\n\n        \n# Setup model, optimizer and criterion\nmodel = riiid_model(settings)\noptimizer = torch.optim.Adam(model.parameters(), lr=.001)\ncriterion = torch.nn.BCEWithLogitsLoss()\nall_auc = []\n\n# Move model and criteriod to device\nmodel.to(settings['device'])\ncriterion.to(settings['device'])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get content_ids and pad\nfor _ in range(5):\n    tbar = tqdm.tqdm(train_dataloader)\n    for batch in tbar:\n        for k in batch.keys():\n            batch[k] = batch[k].to(settings['device'])\n        optimizer.zero_grad()\n        pred = model(batch)\n        loss = criterion(pred, batch['answered_correctly'].float())\n        loss.backward()\n        optimizer.step()\n        \n        # For now, do AUC on only the last prediction\n        t = batch['answered_correctly'][:, -1:][:, -1].detach().to('cpu').numpy()\n        p = pred[:, -1:][:, -1].detach().to('cpu').numpy()\n        auc = sklearn.metrics.roc_auc_score(t, p)\n        all_auc.append(auc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array(all_auc[-200:]).mean())\nmatplotlib.pyplot.plot(all_auc)\nmatplotlib.pyplot.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\nval_ac = np.array([])\nval_pred = np.array([])\n\nfor batch in valid_dataloader:\n    for k in batch.keys():\n        batch[k] = batch[k].to(settings['device'])\n    #optimizer.zero_grad()\n    pred = model(batch)\n    #loss = criterion(pred, batch['answered_correctly'].float())\n    #loss.backward()\n    #optimizer.step()\n\n    # For now, do AUC on only the last prediction\n    t = batch['answered_correctly'][:, -1:][:, -1].detach().to('cpu').numpy()\n    p = pred[:, -1:][:, -1].detach().to('cpu').numpy()\n    \n    # Concatenate\n    val_ac = np.concatenate((val_ac, t))\n    val_pred = np.concatenate((val_pred, p))\n        \nsklearn.metrics.roc_auc_score(val_ac, val_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}