{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nimport gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nimport os\ndef seed_everything(seed = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.set_deterministic(False)\nseed_everything()\n\ndtype = {'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 5, 7], dtype=dtype)\n\ntrain_df = train_df[train_df.content_type_id == False]\ndel train_df['content_type_id']\ngc.collect()\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\n\nquestions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')\ndict_tags = {}\nfor i, tags in enumerate(questions['tags'].unique()):\n    if tags in dict_tags.keys():\n        continue\n    dict_tags[tags] = i\nquestions.tags.replace(dict_tags, inplace=True)\nquestions.part = questions.part.astype(np.int8)\nquestions.tags = questions.tags.astype(np.int16)\nskills = questions[\"question_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))\nquestions.rename(columns = {'question_id': 'content_id'}, inplace=True)\ntrain_df = train_df.merge(questions[['content_id', 'part', 'tags']], on='content_id', how='left')\nskills = questions[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup = train_df.groupby('user_id').apply(lambda r: (\n        r['content_id'].values,\n        r['answered_correctly'].values,\n        r['task_container_id'].values,\n        r['timestamp'].values,\n        r['part'].values,\n        r['tags'].values\n        ))\ndel train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ = 180\nACCEPTED_USER_CONTENT_SIZE = 4\nEMBED_SIZE = 128\nBATCH_SIZE = 64\nDROPOUT = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(shape):\n    future_mask = np.triu(np.ones(shape), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\nclass SubLayer(nn.Module):\n    def __init__(self,embed_dim):\n        super().__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n    def forward(self, e, x):\n        att_mask = future_mask(shape=(e.size(0), x.size(0))).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        return x, att_weight\nclass BERTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):\n        super().__init__()\n        self.pos_embedding = nn.Embedding(max_seq, embed_dim)\n        self.embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.ans_embedding = nn.Embedding(3, embed_dim)\n        self.time_embedding = nn.Embedding(10000, embed_dim)\n        self.lag_time_embedding = nn.Embedding(3600, embed_dim)\n        self.elapsed_time_embedding = nn.Embedding(1520, embed_dim)\n        self.part_embedding = nn.Embedding(8, embed_dim)\n        \n        self.sub1 = SubLayer(embed_dim)\n        \n        self.fc = nn.Linear(embed_dim, embed_dim*2)\n        self.fc1 = nn.Linear(embed_dim*5, embed_dim)\n        \n        self.bacth_norm = nn.BatchNorm1d(max_seq)\n        self.bacth_norm1 = nn.BatchNorm1d(max_seq)\n        \n        self.pred = nn.Linear(embed_dim*2, 1)\n        \n    def forward(self, history_question, history_answer, time, lag_time, part, elapsed_time):\n        device = history_question.device\n        history_answer = history_answer\n        history_answer = self.ans_embedding(history_answer)\n        \n        x = self.embedding(history_question)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        time_x = self.time_embedding(time)\n        lag_time = self.lag_time_embedding(lag_time)\n        part = self.part_embedding(part)\n        elapsed_time = self.elapsed_time_embedding(elapsed_time)\n        \n        history_answer += pos_x\n        x += history_answer\n        time_x += history_answer\n        lag_time += history_answer\n        part += history_answer\n        elapsed_time += history_answer\n        x = torch.cat([x, time_x, lag_time, part, elapsed_time], axis=-1)\n        x = self.fc1(x)\n        \n        \n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        x, att_weight= self.sub1(x, x)\n        \n        x = self.fc(x)\n        x = self.bacth_norm(x)\n\n        x = self.pred(x)\n        \n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = torch.load('/kaggle/input/v9-test/7901_bert_v4.pth',  map_location=lambda storage, loc: storage.cuda(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n        target_task = test_info[\"task_container_id\"]\n        target_timestamp = test_info[\"timestamp\"]\n        target_part = test_info['part']\n        target_tags = test_info['tags']\n        \n    \n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n        task = np.zeros(self.max_seq, dtype=int)\n        lag_time = np.zeros(self.max_seq, dtype=int)\n        part = np.zeros(self.max_seq, dtype=int)\n        tags = np.zeros(self.max_seq, dtype=int)\n        \n        target_lag_time = 0\n        if user_id in self.samples.index:\n            q_, qa_, task_, timestamp_, part_, tags_ = self.samples[user_id]\n            \n            ##获取lag_time\n            lag_time_ = (np.diff(timestamp_)/1000)\n            lag_time_ = lag_time_.astype(int)\n            lag_time_ = np.append([0], lag_time_)\n            lag_time_[lag_time_>3599] = 3599\n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n                task = task_[-self.max_seq:]\n                lag_time = lag_time_[-self.max_seq:]\n                part = part_[-self.max_seq:]\n                tags = tags_[-self.max_seq:]\n                \n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n                task[-seq_len:] = task_\n                lag_time[-seq_len:] = lag_time_\n                part[-seq_len:] = part_\n                tags[-seq_len:] = tags_\n            #print('compute lag time ', target_timestamp, timestamp_[-1])\n            target_lag_time = int((target_timestamp - timestamp_[-1])/1000)\n            #print('target_lag_time ', target_lag_time)\n            if target_lag_time > 3599:\n                target_lag_time = 3599\n        q = np.append(q[1:], [target_id])\n        qa = np.append(qa[1:], [2])\n        task = np.append(task[1:], [target_task])\n        lag_time = np.append(lag_time[1:], [target_lag_time])\n        part = np.append(part[1:], target_part)\n        tags = np.append(tags[1:], target_tags)\n        return q, qa, task, lag_time, part, tags","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\nmodel.eval()\n\n#HDKIM\nprev_test_df = None\n#HDKIMHDKIM\nMAX_SEQ = 180\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    #HDKIM\n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        prev_group = prev_test_df[['timestamp', 'user_id', 'content_id', 'answered_correctly','task_container_id', 'part', 'tags']].groupby('user_id').apply(\n            lambda r:\n            (\n            r['content_id'].values,\n            r['answered_correctly'].values, \n            r['task_container_id'].values,\n            r['timestamp'].values,\n            r['part'].values,\n            r['tags'].values\n            ))\n        for prev_user_id in prev_group.index:\n            prev_group_content, prev_group_ac, prev_group_task,  prev_group_timestamp, prev_part, prev_tags = prev_group[prev_user_id]\n            if prev_user_id in group.index:\n                \n                group[prev_user_id] = (\n                                       np.append(group[prev_user_id][0],prev_group_content), \n                                       np.append(group[prev_user_id][1],prev_group_ac),\n                                       np.append(group[prev_user_id][2],prev_group_task),\n                                       np.append(group[prev_user_id][3],prev_group_timestamp),\n                                       np.append(group[prev_user_id][4],prev_part),\n                                       np.append(group[prev_user_id][5],prev_tags)\n                                      )\n \n            else:\n                group[prev_user_id] = (prev_group_content, prev_group_ac, prev_group_task, prev_group_timestamp, prev_part, prev_tags)\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                new_group_task = group[prev_user_id][2][-MAX_SEQ:]\n                new_group_timestamp = group[prev_user_id][3][-MAX_SEQ:]\n                new_group_part = group[prev_user_id][4][-MAX_SEQ:]\n                new_group_tags = group[prev_user_id][5][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content, new_group_ac, new_group_task, new_group_timestamp, new_group_part, new_group_tags)\n    \n    test_df = test_df.merge(questions[['content_id', 'part', 'tags']], on='content_id', how='left')\n    prev_test_df = test_df.copy()\n    test_df = test_df[test_df.content_type_id == False]\n    #HDKIMHDKIM\n    \n\n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    outs = []\n\n    for item in tqdm(test_dataloader):\n        q = item[0].to(device).long()\n        qa = item[1].to(device).long()\n        task = item[2].to(device).long()\n        lag_time = item[3].to(device).long()\n        part = item[4].to(device).long()\n        tags = item[5].to(device).long()\n        qa[:, -1] = 2\n        with torch.no_grad():\n            output, att_weight = model(q, qa, part, lag_time, part, tags)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        outs.extend(output.view(-1).data.cpu().numpy())\n        \n    test_df['answered_correctly'] =  outs\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}