{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport time\nimport json\nimport psutil\nimport numpy as np\nimport pandas as pd\nimport riiideducation\n\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device='cuda' if torch.cuda.is_available() else 'cpu'\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"MAX_SEQ=100\nMAX_LAG_TIME=2160\nMAX_PREV_ELAPSE_TIME=600\n\nn_questions=13523\nn_parts=7\nn_responses=3\nn_lagtimes=2161\nn_prev_elapsed=601\n\n\nd_model=160\nnhead=8\ndim_feedforward=250\nmax_lr=0.0025","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(torch.utils.data.Dataset):\n    def __init__(self, test_df, max_seq=100):\n        self.test_df=test_df\n        self.max_seq=max_seq\n    def __len__(self):\n        return len(self.test_df)\n    def __getitem__(self, idx):\n        row=self.test_df.iloc[idx]\n        \n        content_id=row.content_id\n        part=row.part\n        timestamp=row.timestamp\n        prior_question_elapsed_time=int(row.prior_question_elapsed_time)\n        prev_seq=row.prev_seq\n        \n        (q_, p_, r_, lag_, prev_elapsed_)=prev_seq\n        seq_len=q_.size\n        \n        q_=torch.tensor(q_, dtype=int)\n        p_=torch.tensor(p_, dtype=int)\n        r_=torch.tensor(r_, dtype=int)\n        lag_=torch.tensor(lag_, dtype=int)\n        prev_elapsed_=torch.tensor(prev_elapsed_, dtype=int)        \n        \n        q=torch.zeros(self.max_seq, dtype=int)\n        p=torch.zeros(self.max_seq, dtype=int)\n        r=torch.zeros(self.max_seq, dtype=int)\n        lag=torch.zeros(self.max_seq, dtype=int)\n        prev_elapsed=torch.zeros(self.max_seq, dtype=int)\n        \n        label_mask=0\n        if seq_len == 0:\n            q[0]=content_id\n            p[0]=part\n            r[0]=2\n            prev_elapsed[0]=prior_question_elapsed_time\n            lag[0]=0\n            label_mask=0\n        elif seq_len == self.max_seq:\n            q[:-1]=q_[1:]\n            p[:-1]=p_[1:]\n            prev_elapsed[:-1]=prev_elapsed_[1:]\n            lag[:-1]=lag_[1:]\n            \n            r[:]=r_[:]\n            q[-1]=content_id\n            p[-1]=part\n            prev_elapsed[-1]=prior_question_elapsed_time\n            lag[-1]=timestamp\n            label_mask=seq_len-1\n        else:\n            q[:seq_len]=q_[:]\n            p[:seq_len]=p_[:]\n            prev_elapsed[:seq_len]=prev_elapsed_[:]\n            lag[:seq_len]=lag_[:]\n            r[1:seq_len+1]=r_[:]\n            \n            q[seq_len]=content_id\n            p[seq_len]=part\n            prev_elapsed[seq_len]=prior_question_elapsed_time\n            lag[seq_len]=timestamp\n            r[0]=2\n            label_mask=seq_len\n            \n        lag[1:]=lag[1:]-lag[:-1]\n        lag=lag/60000\n        lag[lag>1440] =1441+(lag[lag>1440] - 1440)/60\n        lag[0]=0\n        lag[lag<0]=0\n        lag[lag>MAX_LAG_TIME]=MAX_LAG_TIME\n        lag=lag.type(torch.long)\n        \n        return (q, p, r, lag, prev_elapsed, label_mask)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nclass FFN(nn.Module):\n    def __init__(self, d_model=80, dim_feedforward=512, dropout=0.1):\n        super(FFN, self).__init__()\n        self.fc1=nn.Linear(d_model, dim_feedforward)\n        self.relu=nn.ReLU()\n        self.fc2=nn.Linear(dim_feedforward, d_model)\n        self.dropout=nn.Dropout(dropout)\n    def forward(self, x):\n        x=self.fc1(x)\n        x=self.relu(x)\n        x=self.fc2(x)\n        x=self.dropout(x)\n        return x\n    \nclass TransformerLayer(nn.Module):\n    def __init__(self,\n                 d_model=80,\n                 nhead=5,\n                 dim_feedforward=512,\n                 dropout=0.1\n                ):\n        super(TransformerLayer, self).__init__()\n        self.multihead_attn=nn.MultiheadAttention(d_model, \n                                                  num_heads=nhead, \n                                                  dropout=dropout)\n        self.ffn=FFN(d_model, dim_feedforward, dropout)\n        self.layernorm1=nn.LayerNorm(d_model)\n        self.layernorm2=nn.LayerNorm(d_model)\n        \n    def forward(self, Q, K, V, attn_mask=None):\n        Q=self.layernorm1(Q)\n        attn_output, _=self.multihead_attn(Q, K, V, attn_mask=attn_mask)\n        attn_output=Q+attn_output\n        \n        attn_output=self.layernorm2(attn_output)\n        ffn_out=self.ffn(attn_output)\n        layer_out=attn_output + ffn_out\n        return layer_out\n\n\nclass Transformer(nn.Module):\n    def __init__(self, \n                 d_model=80, \n                 nhead=5,\n                 dim_feedforward=512,\n                 num_layers=1,\n                 dropout=0.1\n                ):\n        super(Transformer, self).__init__()\n        self.transformer_layer=TransformerLayer(d_model,nhead,dim_feedforward, dropout)\n        \n    def forward(self, Q, K, V, attn_mask):\n        y=self.transformer_layer(Q, K, V, attn_mask)\n        return y\n    \n    \nclass KTModel(nn.Module):\n    def __init__(self,\n                 n_questions,\n                 n_parts,\n                 n_responses,\n                 n_lagtimes=14400,\n                 n_prev_elapsed=121,\n                 MAX_SEQ=100,\n                 \n                 d_model=80, \n                 nhead=5,\n                 dim_feedforward=512,\n                 num_layers=1,\n                 dropout=0.1,\n                 device='cpu'\n                ):\n        super(KTModel, self).__init__()\n        self.device=device\n        self.pos_ids=torch.arange(MAX_SEQ, device=device)\n        \n        self.pos_embedding=nn.Embedding(MAX_SEQ, d_model)\n        self.q_embedding=nn.Embedding(n_questions, d_model)\n        self.p_embedding=nn.Embedding(n_parts+1, d_model)\n        self.r_embedding=nn.Embedding(n_responses, d_model)\n        self.lag_embedding=nn.Embedding(n_lagtimes, d_model)\n        self.prev_elapsed_embedding=nn.Embedding(n_prev_elapsed, d_model)\n        \n        self.encoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        self.decoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        self.transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)\n        \n        self.dropout=nn.Dropout(dropout)\n        self.out=nn.Linear(d_model, 1)\n    def get_attention_mask(self, sz):\n        attn_mask=torch.tensor(np.triu(np.ones((sz, sz)), k=1).astype('bool'))\n        attn_mask=attn_mask.to(self.device)\n        return attn_mask\n    \n    def get_encoder_inputs(self, q, p):\n        pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))\n        q_embedd=self.dropout(self.q_embedding(q))\n        p_embedd=self.dropout(self.p_embedding(p))\n        encoder_in=pos_embedd+q_embedd+p_embedd\n        return encoder_in\n        \n    def get_decoder_inputs(self, lag_times, prev_elapsed, r):\n        pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))\n        r_embedd=self.dropout(self.r_embedding(r))\n        lag_embedd=self.dropout(self.lag_embedding(lag_times))\n        prev_elapsed_embedd=self.dropout(self.prev_elapsed_embedding(prev_elapsed))\n        decoder_in=pos_embedd+r_embedd+lag_embedd+prev_elapsed_embedd\n        return decoder_in\n    \n    def forward(self, q, p, lag_times, prev_elapsed, r):\n        attn_mask=self.get_attention_mask(q.size(1))\n        encoder_in=self.get_encoder_inputs(q, p)\n        decoder_in=self.get_decoder_inputs(lag_times, prev_elapsed, r)\n        \n\n        encoder_in=encoder_in.permute(1, 0, 2)\n        decoder_in=decoder_in.permute(1, 0, 2)\n        \n        encoder_out=self.encoder_transformer_layer(encoder_in, encoder_in, encoder_in, attn_mask=attn_mask)\n        decoder_out=self.decoder_transformer_layer(decoder_in, decoder_in, decoder_in, attn_mask=attn_mask)\n        \n        y=self.transformer_layer(decoder_out, encoder_out, encoder_out, attn_mask=attn_mask)\n        y=y.permute(1, 0, 2)\n        yout=self.out(y).squeeze(-1)\n        return yout","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=KTModel(n_questions,\n              n_parts,\n              n_responses,\n              n_lagtimes=n_lagtimes,\n              n_prev_elapsed=n_prev_elapsed,\n              MAX_SEQ=MAX_SEQ,\n              d_model=d_model, \n              nhead=nhead,\n              dim_feedforward=dim_feedforward,\n              device=device).to(device)\n\nmodel.load_state_dict(torch.load('../input/saint-v2/sakt_saint (2).pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_group(test_df, prev_test_df):\n    if prev_test_df is None or (psutil.virtual_memory().percent>=90):\n        return\n    prev_answered_correctly=eval(test_df.prior_group_answers_correct.values[0])\n    prev_test_df['answered_correctly']=prev_answered_correctly\n    prev_test_df=prev_test_df[prev_test_df.content_type_id==0]\n    \n    \n    prev_group=prev_test_df.groupby('user_id').apply(lambda row: (row.content_id.values[-MAX_SEQ:],\n                                                                  row.part.values[-MAX_SEQ:],\n                                                                  row.answered_correctly.values[-MAX_SEQ:],\n                                                                  row.timestamp.values[-MAX_SEQ:],\n                                                                  row.prior_question_elapsed_time.values[-MAX_SEQ:]\n                                                                 ))\n    \n    for user_id in prev_group.index.values:\n        if user_id not in group.index:\n            group[user_id]=prev_group[user_id]\n        else:\n            (prev_q, prev_p, prev_r, prev_lag, prev_q_elapsed)=prev_group[user_id]\n            group[user_id]=(\n                np.append(group[user_id][0], prev_q),\n                np.append(group[user_id][1], prev_p),\n                np.append(group[user_id][2], prev_r),\n                np.append(group[user_id][3], prev_lag),\n                np.append(group[user_id][4], prev_q_elapsed)\n            )\n            \n        if len(group[user_id][0]) > MAX_SEQ:\n            new_q=group[user_id][0][-MAX_SEQ:]\n            new_p=group[user_id][1][-MAX_SEQ:]\n            new_r=group[user_id][2][-MAX_SEQ:]\n            new_lag=group[user_id][3][-MAX_SEQ:]\n            new_q_elapsed=group[user_id][4][-MAX_SEQ:]\n            \n            group[user_id]=(new_q, new_p, new_r, new_lag, new_q_elapsed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Group Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprint('Load Group Data')\ngroup=pd.read_pickle('../input/saint-group-submission/saint_group.pkl')\nquestions_df=pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nquestions_df.rename(columns={'question_id': 'content_id'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nprev_test_df=None\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df=test_df[['row_id', 'user_id', 'content_id',  'timestamp',\n                     'content_type_id', 'prior_question_elapsed_time',\n                     'prior_group_answers_correct']].merge(\n        questions_df[['content_id', 'part']],how='left',on='content_id')\n    \n    update_group(test_df, prev_test_df)\n    \n    test_df.part=test_df.part.fillna(5)\n    test_df.prior_question_elapsed_time.fillna(MAX_PREV_ELAPSE_TIME, inplace=True)\n    test_df.prior_question_elapsed_time=test_df.prior_question_elapsed_time/1000\n    test_df.loc[(test_df.prior_question_elapsed_time > MAX_PREV_ELAPSE_TIME), 'prior_question_elapsed_time']=MAX_PREV_ELAPSE_TIME\n    prev_test_df=test_df.copy()\n    \n    \n    test_df=test_df[test_df.content_type_id==0]\n    test_df['prev_seq']=test_df.user_id.apply(lambda user_id: group[user_id] if user_id in group else (np.array([]), np.array([]), np.array([]), np.array([]), np.array([]) ))\n    test_dataset=TestDataset(test_df, max_seq=100)\n    test_dataloader=torch.utils.data.DataLoader(test_dataset,\n                                                batch_size=1024, \n                                                shuffle=False,\n                                                pin_memory=True, \n                                                num_workers=4)\n\n    model.eval()\n    y_answered=[]\n    with torch.no_grad():\n        for (q, p, r, lag, prev_elapsed, label_mask) in test_dataloader:\n            q=q.to(device)\n            p=p.to(device)\n            r=r.to(device)\n            lag=lag.to(device)\n            prev_elapsed=prev_elapsed.to(device)\n            \n            y=model(q, p, lag, prev_elapsed, r)\n            y_answered.extend([torch.sigmoid(y[idx][label_id]).cpu().item() for idx, label_id in enumerate(label_mask)])\n            \n    test_df['answered_correctly']=y_answered\n    env.predict(test_df[['row_id', 'answered_correctly']])\n    del test_df\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}