{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport random\nimport numpy as np \nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n\nfrom torch.optim.lr_scheduler import OneCycleLR, StepLR\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, accuracy_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This notebook is an attempt to include Saint Embeddings into a single transformer block.\n\nthanks to \n1.  https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing/execution\n2. https://www.kaggle.com/leadbest/sakt-with-randomization-state-updates\n3.https://www.kaggle.com/gilfernandes/riiid-self-attention-transformer/data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"device= 'cuda' if torch.cuda.is_available() else 'cpu'\nprint('Device:', device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Constants"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ=100\nMAX_LAG_TIME=2160\nMAX_PREV_ELAPSE_TIME=600\n\nn_questions=13523\nn_parts=7\nn_responses=3\nn_lagtimes=2161\nn_prev_elapsed=601\n\n\nd_model=160\nnhead=8\ndim_feedforward=250\nmax_lr=0.0025\nnum_epochs=1\n\nTRAIN_BATCH_SIZE=128\nVAL_BATCH_SIZE=512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup=pd.read_pickle('../input/riiiidgroupdataset/group.pkl')\ntrain, val=train_test_split(group,test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self,\n                 group,\n                 max_seq=100\n                ):\n        \n        super(Dataset, self).__init__()\n        self.max_seq=max_seq\n        self.group=group\n        self.user_ids=[]\n        self.split_group={}\n        \n        for user_id in self.group.index:\n            (q, p, r, lag, prev_elapsed, _)=self.group[user_id]\n            seq_len=len(q)\n            if seq_len > self.max_seq:\n                last_pos=seq_len//self.max_seq\n                for pos_id in range(last_pos):\n                    start=pos_id * self.max_seq\n                    end=(pos_id+1)*self.max_seq\n                    self.split_group['{}-{}'.format(user_id, pos_id)]=(q[start: end], p[start:end], r[start:end], lag[start: end], prev_elapsed[start:end])\n                    self.user_ids.append('{}-{}'.format(user_id, pos_id))\n                if len(q[end:]) > 5:\n                    self.user_ids.append('{}-{}'.format(user_id, last_pos+1))\n                    self.split_group['{}-{}'.format(user_id, last_pos+1)]=(q[end:], p[end:], r[end:], lag[end:], prev_elapsed[end:])\n            else:\n                self.split_group['{}-0'.format(user_id)]=(q, p, r, lag, prev_elapsed)\n                self.user_ids.append('{}-0'.format(user_id))\n        del self.group\n        gc.collect()\n\n    def __len__(self):\n        return len(self.split_group)\n    \n    def __getitem__(self, idx):\n        user_id=self.user_ids[idx]\n        (q_, p_, r_, lag_, prev_elapsed_)=self.split_group[user_id]\n        \n        seq_len=len(q_)\n        q_=torch.as_tensor(q_, dtype=int)\n        p_=torch.as_tensor(p_, dtype=int)\n        r_=torch.as_tensor(r_, dtype=int)\n        lag_=torch.as_tensor(lag_, dtype=int)\n        prev_elapsed_=torch.as_tensor(prev_elapsed_, dtype=int)\n        \n        q=torch.zeros(self.max_seq, dtype=int)\n        p=torch.zeros(self.max_seq, dtype=int)\n        r=torch.zeros(self.max_seq, dtype=int)\n        y=torch.zeros(self.max_seq, dtype=int)\n        lag=torch.zeros(self.max_seq, dtype=int)\n        prev_elapsed=torch.zeros(self.max_seq, dtype=int)\n        \n        label_mask=torch.zeros(self.max_seq, dtype=bool)\n        label_mask[:seq_len]=True\n        \n        \n        if seq_len < self.max_seq:\n            q[:seq_len]=q_\n            p[:seq_len]=p_\n            y[:seq_len]=r_\n            lag[:seq_len]=lag_\n            prev_elapsed[:seq_len]=prev_elapsed_\n            \n            r[0]=2\n            r[1:seq_len]=r_[:seq_len-1]\n        else:\n            q[:]=q_[:self.max_seq]\n            p[:]=p_[:self.max_seq]\n            y[:]=r_[:self.max_seq]\n            lag[:]=lag_[:self.max_seq]\n            prev_elapsed[:]=prev_elapsed_[:self.max_seq]\n            r[0]=2\n            r[1:]=r_[: self.max_seq-1]\n        \n        if seq_len>1:\n            lag[1:]=lag[1:]-lag[:-1]\n            lag=lag/60000\n            lag[lag>1440] =1441+(lag[lag>1440] - 1440)/60\n            \n        lag[0]=0\n        lag[lag<0]=0\n        lag[lag>MAX_LAG_TIME]=MAX_LAG_TIME\n        lag=lag.type(torch.long)\n        return (q, p, r, lag, prev_elapsed, y, label_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_dataset=Dataset(train, max_seq=MAX_SEQ)\nval_dataset=Dataset(val, max_seq=MAX_SEQ)\n\n\n\ntrain_dataloader=torch.utils.data.DataLoader(train_dataset, \n                                             batch_size=TRAIN_BATCH_SIZE,\n                                             shuffle=True, \n                                             pin_memory=True,\n                                             num_workers=8)\n\nval_dataloader=torch.utils.data.DataLoader(val_dataset, \n                                           batch_size=VAL_BATCH_SIZE, \n                                           shuffle=False,\n                                           pin_memory=True,\n                                           num_workers=8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nclass FFN(nn.Module):\n    def __init__(self, d_model=80, dim_feedforward=512, dropout=0.1):\n        super(FFN, self).__init__()\n        self.fc1=nn.Linear(d_model, dim_feedforward)\n        self.relu=nn.ReLU()\n        self.fc2=nn.Linear(dim_feedforward, d_model)\n        self.dropout=nn.Dropout(dropout)\n    def forward(self, x):\n        x=self.fc1(x)\n        x=self.relu(x)\n        x=self.fc2(x)\n        x=self.dropout(x)\n        return x\n    \nclass TransformerLayer(nn.Module):\n    def __init__(self,\n                 d_model=80,\n                 nhead=5,\n                 dim_feedforward=512,\n                 dropout=0.1\n                ):\n        super(TransformerLayer, self).__init__()\n        self.multihead_attn=nn.MultiheadAttention(d_model, \n                                                  num_heads=nhead, \n                                                  dropout=dropout)\n        self.ffn=FFN(d_model, dim_feedforward, dropout)\n        self.layernorm1=nn.LayerNorm(d_model)\n        self.layernorm2=nn.LayerNorm(d_model)\n        \n    def forward(self, Q, K, V, attn_mask=None):\n        Q=self.layernorm1(Q)\n        attn_output, _=self.multihead_attn(Q, K, V, attn_mask=attn_mask)\n        attn_output=Q+attn_output\n        \n        attn_output=self.layernorm2(attn_output)\n        ffn_out=self.ffn(attn_output)\n        layer_out=attn_output + ffn_out\n        return layer_out\n\n\nclass Transformer(nn.Module):\n    def __init__(self, \n                 d_model=80, \n                 nhead=5,\n                 dim_feedforward=512,\n                 num_layers=1,\n                 dropout=0.1\n                ):\n        super(Transformer, self).__init__()\n        self.transformer_layer=TransformerLayer(d_model,nhead,dim_feedforward, dropout)\n        \n    def forward(self, Q, K, V, attn_mask):\n        y=self.transformer_layer(Q, K, V, attn_mask)\n        return y\n    \n    \nclass KTModel(nn.Module):\n    def __init__(self,\n                 n_questions,\n                 n_parts,\n                 n_responses,\n                 n_lagtimes=14400,\n                 n_prev_elapsed=121,\n                 MAX_SEQ=100,\n                 \n                 d_model=80, \n                 nhead=5,\n                 dim_feedforward=512,\n                 num_layers=1,\n                 dropout=0.1,\n                 device='cpu'\n                ):\n        super(KTModel, self).__init__()\n        self.device=device\n        self.pos_ids=torch.arange(MAX_SEQ, device=device)\n        \n        self.pos_embedding=nn.Embedding(MAX_SEQ, d_model)\n        self.q_embedding=nn.Embedding(n_questions, d_model)\n        self.p_embedding=nn.Embedding(n_parts+1, d_model)\n        self.r_embedding=nn.Embedding(n_responses, d_model)\n        self.lag_embedding=nn.Embedding(n_lagtimes, d_model)\n        self.prev_elapsed_embedding=nn.Embedding(n_prev_elapsed, d_model)\n        \n        self.encoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        self.decoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        self.transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        \n        self.dropout=nn.Dropout(dropout)\n        self.out=nn.Linear(d_model, 1)\n    def get_attention_mask(self, sz):\n        attn_mask=torch.tensor(np.triu(np.ones((sz, sz)), k=1).astype('bool'))\n        attn_mask=attn_mask.to(self.device)\n        return attn_mask\n    \n    def get_encoder_inputs(self, q, p):\n        pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))\n        q_embedd=self.dropout(self.q_embedding(q))\n        p_embedd=self.dropout(self.p_embedding(p))\n        encoder_in=pos_embedd+q_embedd+p_embedd\n        return encoder_in\n        \n    def get_decoder_inputs(self, lag_times, prev_elapsed, r):\n        pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))\n        r_embedd=self.dropout(self.r_embedding(r))\n        lag_embedd=self.dropout(self.lag_embedding(lag_times))\n        prev_elapsed_embedd=self.dropout(self.prev_elapsed_embedding(prev_elapsed))\n        decoder_in=pos_embedd+r_embedd+lag_embedd+prev_elapsed_embedd\n        return decoder_in\n    \n    def forward(self, q, p, lag_times, prev_elapsed, r):\n        attn_mask=self.get_attention_mask(q.size(1))\n        encoder_in=self.get_encoder_inputs(q, p)\n        decoder_in=self.get_decoder_inputs(lag_times, prev_elapsed, r)\n        \n\n        encoder_in=encoder_in.permute(1, 0, 2)\n        decoder_in=decoder_in.permute(1, 0, 2)\n        \n        encoder_out=self.encoder_transformer_layer(encoder_in, encoder_in, encoder_in, attn_mask=attn_mask)\n        decoder_out=self.decoder_transformer_layer(decoder_in, decoder_in, decoder_in, attn_mask=attn_mask)\n        \n        y=self.transformer_layer(decoder_out, encoder_out, encoder_out, attn_mask=attn_mask)\n        y=y.permute(1, 0, 2)\n        yout=self.out(y).squeeze(-1)\n        return yout","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef train_epoch(model, optim, criterion, schedular=None):\n    train_loss=[]\n    model.train()\n    for i, (q, p, r,lag, prev_elapsed, y, label_mask) in enumerate(train_dataloader):\n        q=q.to(device)\n        p=p.to(device)\n        lag=lag.to(device)\n        prev_elapsed=prev_elapsed.to(device)\n        r=r.to(device)\n        y=y.to(device)\n        \n        label_mask=label_mask.to(device)\n        \n        optim.zero_grad()\n        yout=model(q, p, lag,prev_elapsed, r)\n        \n        y=torch.masked_select(y, label_mask).type(torch.cuda.FloatTensor if device == 'cuda' else torch.FloatTensor)\n        yout=torch.masked_select(yout, label_mask).to(device)\n        \n        loss_=criterion(yout, y)\n        loss_.backward()\n        optim.step()\n        if schedular is not None:\n            schedular.step()\n        train_loss.append(loss_.item())\n    return np.mean(train_loss)\n\ndef val_epoch(model, criterion):\n    val_loss=[]\n    ytrue=[]\n    ypred=[]\n    \n    model.eval()\n    for (q, p, r, lag, prev_elapsed, y, label_mask) in val_dataloader:\n        q=q.to(device)\n        p=p.to(device)\n        r=r.to(device)\n        lag=lag.to(device)\n        prev_elapsed=prev_elapsed.to(device)\n        y=y.to(device)\n        \n        with torch.no_grad():\n            yout=model(q, p, lag,prev_elapsed, r)\n            y=torch.masked_select(y, label_mask).type(torch.cuda.FloatTensor if device == 'cuda' else torch.FloatTensor)\n            yout=torch.masked_select(yout, label_mask).to(device)\n            loss_=criterion(yout, y)\n            val_loss.append(loss_.item())\n            \n            ytrue.extend(y.cpu().numpy())\n            ypred.extend(torch.sigmoid(yout).cpu().numpy())\n    roc_score=roc_auc_score(ytrue, ypred)\n    return (np.mean(val_loss), roc_score)\n\ndef train_model(num_epochs, max_lr, model):\n    best_loss=None\n    epoch_train_loss=[]\n    epoch_val_loss=[]\n    \n    optim=torch.optim.Adam(model.parameters(), lr=max_lr)\n    criterion=torch.nn.BCEWithLogitsLoss().to(device)\n\n    schedular=OneCycleLR(optim, \n                     max_lr=max_lr,\n                     steps_per_epoch=len(train_dataloader),\n                     epochs=num_epochs)\n\n    for i in range(num_epochs):\n        start_time=time.time()\n        train_loss_=train_epoch(model, optim, criterion, schedular)\n        (val_loss_, roc_score)=val_epoch(model, criterion)\n\n        epoch_train_loss.append(train_loss_)\n        epoch_val_loss.append(val_loss_)\n\n        if best_loss==None or best_loss>val_loss_:\n            best_loss=val_loss_\n            torch.save(model.state_dict(), 'sakt_saint.pth')\n\n        end_time=time.time()\n        print(\"--------------------------------\")\n        print(\"Epoch:{} | Train Loss:{:.4f} | Val Loss:{:4f} | Val ROC-Score:{:.4f}\".format(i, train_loss_, val_loss_, roc_score))\n        print('Epoch Time: {:.4f}'.format(end_time-start_time))\n        print()\n        gc.collect()\n    return (epoch_train_loss, epoch_val_loss)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nmodel=KTModel(n_questions,\n              n_parts,\n              n_responses,\n              n_lagtimes=n_lagtimes,\n              n_prev_elapsed=n_prev_elapsed,\n              MAX_SEQ=MAX_SEQ,\n              d_model=d_model, \n              nhead=nhead,\n              dim_feedforward=dim_feedforward,\n              device=device).to(device)\n\n(epoch_train_loss1, epoch_val_loss1) = train_model(15, 0.0025,model)\n(epoch_train_loss2, epoch_val_loss2) = train_model(5, 0.002,model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('Train Loss')\nplt.plot(epoch_val_loss1)\nplt.show()\n\nplt.title('Val Loss')\nplt.plot(epoch_val_loss1)\nplt.show()\nprint('Exit...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('Train Loss')\nplt.plot(epoch_val_loss2)\nplt.show()\n\nplt.title('Val Loss')\nplt.plot(epoch_val_loss2)\nplt.show()\nprint('Exit...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}