{"cells":[{"metadata":{},"cell_type":"markdown","source":"###### * Base Source: https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n* My First Work: https://www.kaggle.com/leadbest/sakt-self-attentive-knowledge-tracing-submitter\n\n1. Version 1: State Updates -> LB 0.765\n2. Version 3: Random Selection of User Interactions -> LB 0.768\n3. Version 6: Small Optimization -> LB 0.771?"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM\nMAX_SEQ = 160\n#HDKIMHDKIM","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndtype = {'row_id': 'int64','timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8','prior_question_elapsed_time': 'float32'}\n\ndata_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[0,1, 2, 3, 4, 7,8], dtype=dtype)\ndata_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_df = data_df[data_df.content_type_id == False] #ojo con esto que yp lo tengo como 0 y 1 el flag\n\n#arrange by timestamp\ndata_df = data_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"skills = data_df[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"group = data_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n\ndel data_df\ngc.collect()"},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM\nimport random\nrandom.seed(1)\n#HDKIMHDKIM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold,train_test_split\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#    unique_indexes= y_train.groupby('user_id_x').mean()\n\n#     #aux_df = pd.concat([X_train[\"variable\"],y_train],axis=1)\n\n#   #  for fold,(train_index, test_index) in enumerate(mkf.split(unique_indexes,unique_indexes)):\n     \n#     for fold,(train_index, val_index) in enumerate(mkf.split(unique_indexes,pd.qcut(unique_indexes,5,labels = [i for i in range(5)],duplicate='drop'))):","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reshape_to_accomodate_max_len(x,cut_len =160):\n    \n    # This assumes that you have a 2-dimensional array\n    #zeros = np.zeros(shape, dtype=np.int32)\n   # zeros[:sigma.shape[0], :sigma.shape[1]] = sigma\n    \n    \n    if x.shape[0] >cut_len:\n        cuts = np.arange(0,x.shape[0],cut_len) if  ( x.shape[0] % cut_len ) else  np.arange(0,x.shape[0],cut_len).tolist() +[x.shape[0]]\n        return np.array_split(x,cuts)\n    else:\n        return [x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lag_time(df):\n    \n    df.set_index('row_id',inplace = True)\n\n    \n    \n    \n    return df['timestamp'].diff() - df['prior_question_elapsed_time']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lag_time\nres_lag_time = data_df[['row_id','user_id','timestamp','prior_question_elapsed_time']].groupby('user_id').apply(get_lag_time )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prior_question_elapsed_time\n\n#prior_question_elapsed_time is made with a shift from the prior","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup = data_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: \n            reshape_to_accomodate_max_len(r[['content_id', 'answered_correctly']].values,cut_len=640)).explode().apply(lambda x: np.array(x).transpose())\n\n\ndel data_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_indexes= group.apply(lambda x: len(x[0]) )\n\n\ngroup_train,group_val = train_test_split(group,test_size=0.1,stratify = pd.qcut(unique_indexes,5,labels = [i for i in range(5)]),random_state=14)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_train.reset_index(drop=True,inplace=True)\ngroup_val.reset_index(drop=True,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = group.groupby('user_id').tail(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(group.apply(lambda x:len(x[0])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group.loc[(group.apply(lambda x:len(x[0])) ==0)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = group.loc[(group.apply(lambda x:len(x[0])) !=0)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle as pk\n\npk.dump(group,open('group.pkl','wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del group\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SAKTDataset(Dataset):\n    def __init__(self, group, n_skill, max_seq=MAX_SEQ): #HDKIM 100\n        super(SAKTDataset, self).__init__()\n        self.max_seq = max_seq\n        self.n_skill = n_skill\n        self.samples = group\n\n        #         self.user_ids = [x for x in group.index]\n        self.user_ids = []\n        for user_id in group.index:\n            q, qa = group[user_id]\n            if len(q) < 2: #HDKIM 10\n                 continue\n            self.user_ids.append(user_id)\n\n        #HDKIM Memory reduction\n        #if len(q)>self.max_seq:\n        #    group[user_id] = (q[-self.max_seq:],qa[-self.max_seq:])\n\n    def __len__(self):\n        return len(self.user_ids)\n        #return len(self.samples)\n\n    def __getitem__(self, index):\n        user_id = self.user_ids[index]\n        q_, qa_ = self.samples[user_id]\n        #q_, qa_ = self.samples.iloc[index]\n        seq_len = len(q_)\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        \n        if seq_len >= self.max_seq:\n            #HDKIM\n            if random.random()>0.02:\n                start = random.randint(0,(seq_len-self.max_seq))\n                end = start + self.max_seq\n                q[:] = q_[start:end]\n                qa[:] = qa_[start:end]\n            else:\n                #HDKIMHDKIM\n                q[:] = q_[-self.max_seq:]\n                qa[:] = qa_[-self.max_seq:]\n        else:\n            #HDKIM\n            if random.random()>0.02:\n                #HDKIMHDKIM\n                start = 0\n                end = random.randint(2,seq_len)\n                seq_len = end - start\n                q[-seq_len:] = q_[0:seq_len]\n                qa[-seq_len:] = qa_[0:seq_len]\n            else:\n                #HDKIMHDKIM\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_\n\n        \n        target_id = q[1:]\n        label = qa[1:]\n\n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[:-1].copy()\n        x += (qa[:-1] == 1) * self.n_skill\n\n        return x, target_id, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = SAKTDataset(group_train, n_skill)\nval_dataset = SAKTDataset(group_val, n_skill)\n\n#dataloader = DataLoader(dataset, batch_size=2048, shuffle=True, num_workers=8)\n\nitem = train_dataset.__getitem__(5)\n# print(item[0])\n# print(item[1])\n# print(item[2])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nclass PositionalEncoding(nn.Module):\n    r\"\"\"Inject some information about the relative or absolute position of the tokens\n        in the sequence. The positional encodings have the same dimension as\n        the embeddings, so that the two can be summed. Here, we use sine and cosine\n        functions of different frequencies.\n    .. math::\n        \\text{PosEncoder}(pos, 2i) = sin(pos/10000^(2i/d_model))\n        \\text{PosEncoder}(pos, 2i+1) = cos(pos/10000^(2i/d_model))\n        \\text{where pos is the word position and i is the embed idx)\n    Args:\n        d_model: the embed dim (required).\n        dropout: the dropout value (default=0.1).\n        max_len: the max. length of the incoming sequence (default=5000).\n    Examples:\n        >>> pos_encoder = PositionalEncoding(d_model)\n    \"\"\"\n\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        \n        #position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n       # position =torch.repeat_interleave(torch.arange(0, max_len  // tokens_per_word , dtype=torch.float), tokens_per_word).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        r\"\"\"Inputs of forward function\n        Args:\n            x: the sequence fed to the positional encoder model (required).\n        Shape:\n            x: [sequence length, batch size, embed dim]\n            output: [sequence length, batch size, embed dim]\n        Examples:\n            >>> output = pos_encoder(x)\n        \"\"\"\n\n        return self.pe[:x.size(0), :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass PosSAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        \n        #self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.pos_embedding = PositionalEncoding(max_len=max_seq-1,d_model=embed_dim)   #nn.Embedding(max_seq-1, embed_dim)\n        \n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        \n       # print(x.size(0),x.size(1),x.size(2))\n       # print(x.size(1))\n        \n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        \n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTmodule(nn.Module):\n    def __init__(self,  embed_dim=128,num_heads=8,dropout=0.1): #HDKIM 100->MAX_SEQ\n        super(SAKTmodule, self).__init__()\n        \n       # self.att_mask = att_mask\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, dropout= dropout)\n        self.self_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, dropout= dropout)\n\n        \n        \n        self.layer_normal_self1 = nn.LayerNorm(embed_dim) \n        self.layer_normal_self2 = nn.LayerNorm(embed_dim) \n        \n        self.layer_normal1 = nn.LayerNorm(embed_dim) \n        self.layer_normal2 = nn.LayerNorm(embed_dim) \n\n        self.ffn1 = FFN(embed_dim)\n        self.ffn2 = FFN(embed_dim)\n        \n       \n    \n    def forward(self, x, e,att_mask):\n       \n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        \n        selfatt_output, selfatt_mask = self.self_att(e, e, e, attn_mask= att_mask)\n        \n       \n        e = self.layer_normal_self1(selfatt_output + e) #se suma esto ahora?\n        \n        e = e.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n         \n        e1 = self.ffn1(e)\n        \n        e = self.layer_normal_self2(e1 + e)\n        \n        e = e.permute(1, 0, 2)\n        \n        att_output, att_weight = self.multi_att(e, x, x, attn_mask= att_mask)\n        \n        e = self.layer_normal1(att_output + e)\n        \n        e = e.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn2(e)\n        x = self.layer_normal2(x + e)\n        \n        \n        \n\n        return x, e ,att_mask\n    \n    \n\n\n\n\n\n\n\n\nclass SSAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SSAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.SAKT1 = SAKTmodule(embed_dim=embed_dim)\n        self.SAKT2 = SAKTmodule(embed_dim=embed_dim)\n        self.SAKT3 = SAKTmodule(embed_dim=embed_dim)\n\n        \n        \n        self.dropout = nn.Dropout(0.2)\n        \n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n        \n        x = x.permute(1, 0, 2)\n        \n        att_mask = future_mask(x.size(0)).to(device)\n        \n        x = x.permute(1, 0, 2)\n        \n        x, e,att_mask = self.SAKT1(x, e,att_mask)\n        x, e,att_mask = self.SAKT2(x, e,att_mask)\n        x, e,att_mask= self.SAKT3(x, e,att_mask)\n    \n        x = self.pred(x)\n        \n        \n\n        return x.squeeze(-1),att_mask\n    \n    \n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n#model = SAKTModel(n_skill, embed_dim=128)\n# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)\n#optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n#criterion = nn.BCEWithLogitsLoss()\n\n#model.to(device)\n#criterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):\n    model.train()\n    final_loss = 0\n    \n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n        optimizer.zero_grad()\n        \n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n      \n        output, _ = model(x, target_id)\n        \n        loss = loss_fn(output, label)\n        loss.backward()\n        \n        optimizer.step()\n        \n        if not  scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n            scheduler.step()\n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    \n    return final_loss\n\n\ndef valid_fn(model, scheduler, loss_fn, dataloader, device):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    labels = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n       \n        \n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n      \n        output, _ = model(x, target_id)\n        \n        loss = loss_fn(output, label)\n        \n        \n        output = torch.sigmoid(output)\n        \n         \n        output = output[:, -1]\n        label = label[:, -1] \n        \n        \n        labels.extend(label.view(-1).data.cpu().numpy())\n        valid_preds.extend(output.view(-1).data.cpu().numpy())\n       \n        \n        #valid_preds.append(output.sigmoid().detach().cpu().numpy())\n       # labels.append(label.detach().cpu().numpy()[0])\n        \n        if scheduler.__class__ ==  torch.optim.lr_scheduler.ReduceLROnPlateau:\n            scheduler.step(final_loss)\n    \n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    auc = roc_auc_score(labels, valid_preds)\n    \n    return final_loss,valid_preds,auc\n    \n\ndef inference_fn(model, dataloader, device):\n    model.eval()\n    outs = []\n    \n    tbar = tqdm(dataloader)\n    \n    for item in tbar:\n        optimizer.zero_grad()\n        \n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        with torch.no_grad():\n            output, _ = model(x, target_id)\n            \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n        \n        outs.extend(output.data.cpu().numpy())\n    \n    return outs\n\n\ndef infer_features_fn(model, dataloader, device):\n    model.eval()\n    preds = []\n    \n    for item in dataloader:\n        inputs = data['x'].to(device)\n\n        with torch.no_grad():\n            outputs = model(inputs,mode='get_features')\n        \n#         print(len(outputs))\n        \n#         for i in range(len(outputs)):\n#             print(outputs[i].shape)\n            \n#         print(torch.cat(outputs,axis=1).shape)\n        \n        preds.append(torch.cat(outputs,axis=1).detach().cpu().numpy())\n        \n        \n        \n    preds = np.concatenate(preds)\n    \n    return preds\n   \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy_score():\n    \n    pred = (torch.sigmoid(output) >= 0.5).long()\n\n    num_corrects += (pred == label).sum().item()\n    num_total += len(label)\n\n    auc = roc_auc_score(labels, outs)\n    acc = num_corrects / num_total\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#chequear q quc no sea solo en ultima pregunta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nEPOCHS = 40\n\nLR = 6e-3\n\nBATCH_SIZE = 256\n\nEARLY_STOPPING_STEPS = 5\n\nEARLY_STOP = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    #os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import _LRScheduler\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NoamLR(_LRScheduler):\n    \"\"\"\n    Implements the Noam Learning rate schedule. This corresponds to increasing the learning rate\n    linearly for the first ``warmup_steps`` training steps, and decreasing it thereafter proportionally\n    to the inverse square root of the step number, scaled by the inverse square root of the\n    dimensionality of the model. Time will tell if this is just madness or it's actually important.\n    Parameters\n    ----------\n    warmup_steps: ``int``, required.\n        The number of steps to linearly increase the learning rate.\n    \"\"\"\n    def __init__(self, optimizer, warmup_steps):\n        self.warmup_steps = warmup_steps\n        super().__init__(optimizer)\n\n    def get_lr(self):\n        last_epoch = max(1, self.last_epoch)\n        scale = self.warmup_steps ** 0.5 * min(last_epoch ** (-0.5), last_epoch * self.warmup_steps ** (-1.5))\n        return [base_lr * scale for base_lr in self.base_lrs]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    \ndef train_pipeline(train_dataset,valid_dataset, seed,fold,verbose=False,**kwargs):  \n    \n    seed_everything(seed)\n    \n   \n    trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n   \n    model = SSAKTModel(n_skill, embed_dim=128)\n    \n    model.to(DEVICE)\n    \n    #initialize_from_past_model(model, f\"../results/original_torch_moa_smoothed_lrplateau_5_folds_AUX_SEED{seed}_FOLD{fold}.pth\")#,freeze_first_layer=True)\n    \n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n    #optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n    #scheduler = optim.lr_scheduler.OneCycleLR(optimizer, pct_start=0.1, div_factor=1e3, \n     #                                         max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))\n    \n    scheduler = NoamLR(optimizer,12000) #optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)\n    \n    \n    \n    loss_val = nn.BCEWithLogitsLoss().to(DEVICE)\n    loss_tr =  nn.BCEWithLogitsLoss().to(DEVICE)\n    \n    \n    early_stopping_steps = EARLY_STOPPING_STEPS\n    early_step = 0\n    \n    #todo el guardado de los resultados se puede mover a kfold que si tiene info de los indices\n    #oof = np.zeros((len(train), target.iloc[:, 1:].shape[1]))\n    best_loss = np.inf\n    \n    best_auc = 0\n    \n    \n    \n    \n    for epoch in range(EPOCHS):\n        \n        train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)\n        if verbose:\n            print(f\"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}, lr:{scheduler.get_last_lr()[0]}\")\n        valid_loss, valid_preds ,auc = valid_fn(model,scheduler, loss_val, validloader, DEVICE)\n        if verbose:\n            print(f\"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}, auc: {auc} \")\n\n    #         if valid_loss < best_loss:\n\n    #             best_loss = valid_loss\n    #             oof = valid_preds\n    #             torch.save(model.state_dict(), \"SAKT-HDKIM.pt\")\n\n        if auc > best_auc:\n\n            best_auc = auc\n            oof = valid_preds\n            torch.save(model.state_dict(), \"SAKT-HDKIM.pt\")\n\n\n\n       #     torch.save(model.state_dict(), f\"../results/{exp_name}_SEED{seed}_FOLD{fold}.pth\")\n\n        elif(EARLY_STOP == True):\n\n            early_step += 1\n            if (early_step >= early_stopping_steps):\n                break\n\n    \n    #--------------------- PREDICTION---------------------\n   \n  #  testdataset = TestDataset(X_test)\n  #  testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n#     model = Model(\n#          num_features= X_train.shape[1] ,\n#         num_targets=  y_train.shape[1],\n#         hidden_size=hidden_size,**kwargs\n#     )\n    \n#     model.load_state_dict(torch.load(f\"../results/FOLD{fold}_{exp_name}.pth\"))\n   # model.to(DEVICE)\n    \n    #predictions = np.zeros((len(test_), target.iloc[:, 1:].shape[1]))\n    \n    #predictions = inference_fn(model, testloader, DEVICE)\n    \n    return oof\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" oof = train_pipeline(train_dataset,val_dataset,42,fold=0,verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def train_epoch(model, train_iterator, optim, criterion, device=\"cpu\"):\n    model.train()\n\n    train_loss = []\n    num_corrects = 0\n    num_total = 0\n    labels = []\n    outs = []\n    \n    \n    \n    tr_loss = train_fn()\n    \n    \n\n    tbar = tqdm(train_iterator)\n    for item in tbar:\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n        label = item[2].to(device).float()\n\n        optim.zero_grad()\n        output, atten_weight = model(x, target_id)\n        loss = criterion(output, label)\n        loss.backward()\n        optim.step()\n        train_loss.append(loss.item())\n        \n        \n        \n        trI\n       \n        output = output[:, -1]\n        label = label[:, -1] \n        \n        \n        labels.extend(label.view(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n\n        #accuracy\n        pred = (torch.sigmoid(output) >= 0.5).long()\n        \n        num_corrects += (pred == label).sum().item()\n        num_total += len(label)\n\n        tbar.set_description('loss - {:.4f}'.format(loss))\n\n    acc = num_corrects / num_total\n    auc = roc_auc_score(labels, outs)\n    loss = np.mean(train_loss)\n\n    return loss, acc, auc"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"epochs = 35 #HDKIM 20\nfor epoch in range(epochs):\n    \n    loss, acc, auc = train_epoch(model, dataloader, optimizer, criterion, device)\n    print(\"epoch - {} train_loss - {:.2f} acc - {:.3f} auc - {:.3f}\".format(epoch, loss, acc, auc))"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train_dataset,val_dataset\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"group = pk.load(open('group.pkl','rb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = SSAKTModel(n_skill, embed_dim=128)\n\nmodel.load_state_dict(torch.load(f\"SAKT-HDKIM.pt\",map_location=device))\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\nmodel.eval()\n\n#HDKIM\nprev_test_df = None\n#HDKIMHDKIM\n\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n        print(psutil.virtual_memory().percent)\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_ac = prev_group[prev_user_id][1]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_content), \n                                       np.append(group[prev_user_id][1],prev_group_ac))\n \n            else:\n                group[prev_user_id] = (prev_group_content,prev_group_ac)\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content,new_group_ac)\n\n    prev_test_df = test_df.copy()\n \n    #HDKIMHDKIM\n    \n    test_df = test_df[test_df.content_type_id == False]\n                \n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    outs = []\n\n    for item in tqdm(test_dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = model(x, target_id)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        # pred = (output >= 0.5).long()\n        # loss = criterion(output, label)\n\n        # val_loss.append(loss.item())\n        # num_corrects += (pred == label).sum().item()\n        # num_total += len(label)\n\n        # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] =  outs\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}