{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:34.658366Z","iopub.status.busy":"2021-01-05T12:31:34.657597Z","iopub.status.idle":"2021-01-05T12:31:35.557147Z","shell.execute_reply":"2021-01-05T12:31:35.558026Z"},"papermill":{"duration":0.916133,"end_time":"2021-01-05T12:31:35.55819","exception":false,"start_time":"2021-01-05T12:31:34.642057","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import riiideducation\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport os\nimport sklearn\n\nprint(sklearn.__version__)\n\nuser_cache = dict()\n\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:35.587152Z","iopub.status.busy":"2021-01-05T12:31:35.586546Z","iopub.status.idle":"2021-01-05T12:31:35.621499Z","shell.execute_reply":"2021-01-05T12:31:35.622001Z"},"papermill":{"duration":0.052463,"end_time":"2021-01-05T12:31:35.622124","exception":false,"start_time":"2021-01-05T12:31:35.569661","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"questions_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\").set_index(\"question_id\")\n\nqid_max = questions_df.index.max() + 1\n\nquestions_df[\"part\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:35.650568Z","iopub.status.busy":"2021-01-05T12:31:35.649946Z","iopub.status.idle":"2021-01-05T12:31:35.691181Z","shell.execute_reply":"2021-01-05T12:31:35.690687Z"},"papermill":{"duration":0.058421,"end_time":"2021-01-05T12:31:35.691278","exception":false,"start_time":"2021-01-05T12:31:35.632857","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nlectures_df = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/lectures.csv\")\nle = LabelEncoder()\nle.fit(lectures_df[\"lecture_id\"].values)\n\ndef encode_cid(x):\n    return qid_max # + le.transform(x)\nlectures_df[\"part\"] = 8\n\nlectures_df[\"lecture_id\"] = encode_cid(lectures_df[\"lecture_id\"])\nlectures_df.set_index(\"lecture_id\", inplace=True)\nlid_max = lectures_df.index.max() + 1\n\nlectures_df[\"part\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:35.724299Z","iopub.status.busy":"2021-01-05T12:31:35.723493Z","iopub.status.idle":"2021-01-05T12:31:35.727177Z","shell.execute_reply":"2021-01-05T12:31:35.726687Z"},"papermill":{"duration":0.024814,"end_time":"2021-01-05T12:31:35.727272","exception":false,"start_time":"2021-01-05T12:31:35.702458","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"lpart_dict = lectures_df[\"part\"].to_dict()\nqpart_dict = questions_df[\"part\"].to_dict()\n\nlen(lpart_dict), len(qpart_dict)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:35.754619Z","iopub.status.busy":"2021-01-05T12:31:35.753856Z","iopub.status.idle":"2021-01-05T12:31:35.757534Z","shell.execute_reply":"2021-01-05T12:31:35.75707Z"},"papermill":{"duration":0.019176,"end_time":"2021-01-05T12:31:35.757631","exception":false,"start_time":"2021-01-05T12:31:35.738455","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"CA = questions_df[\"correct_answer\"].values\nCA","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:35.823194Z","iopub.status.busy":"2021-01-05T12:31:35.805674Z","iopub.status.idle":"2021-01-05T12:31:44.385318Z","shell.execute_reply":"2021-01-05T12:31:44.384827Z"},"papermill":{"duration":8.614601,"end_time":"2021-01-05T12:31:44.385421","exception":false,"start_time":"2021-01-05T12:31:35.77082","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch\nfrom collections import OrderedDict\n\n\nEMB_DIM = 8\n\n\ndef l2norm(q):\n    qn = torch.norm(q, p=2, dim=2).detach()\n    qn = qn.unsqueeze(-1).repeat(1, 1, q.shape[2])\n    q = q.div(qn)\n    return q\n\n\nclass RidModel(nn.Module):\n    def __init__(self, gru_dim=128, emb_dim=EMB_DIM):\n        super(RidModel, self).__init__()\n        self.content_difficulty_emb = nn.Embedding(lid_max, embedding_dim=2)\n        self.content_answer_emb = nn.Embedding(lid_max, embedding_dim=4)\n        self.answer_emb = nn.Embedding(5, embedding_dim=4)\n        \n        self.num_heads = 8\n        self.content_emb_size = 184\n        total_emb_size = self.content_emb_size + emb_dim\n        self.content_emb = nn.Embedding(lid_max, embedding_dim=self.content_emb_size)\n        self.sim_convs = nn.ModuleList([nn.Linear(total_emb_size, 32) for i in range(self.num_heads)])\n        \n        self.part_emb = nn.Embedding(lectures_df.part.max() + 1, embedding_dim=emb_dim)\n        self.hidden = nn.Sequential(nn.Linear(10 + self.num_heads + total_emb_size, gru_dim), nn.Tanh())\n        self.gate = nn.Sequential(nn.Linear(10 + self.num_heads + total_emb_size, gru_dim), nn.Sigmoid())\n        \n        self.inv_gru = nn.GRU(gru_dim, gru_dim, batch_first=True)\n        self.gru = nn.GRU(2*gru_dim, gru_dim, batch_first=True)\n        \n        self.inv_gru2 = nn.GRU(gru_dim, gru_dim, batch_first=True)\n        self.gru2 = nn.GRU(2*gru_dim, gru_dim, batch_first=True)\n        self.final = nn.Sequential(nn.Linear(2 + gru_dim + total_emb_size + 3, 128),\n                                   nn.BatchNorm1d(128),\n                                   nn.ReLU(),\n                                   nn.Linear(128, 32),\n                                   nn.BatchNorm1d(32),\n                                   nn.ReLU(),\n                                   nn.Linear(32, 1))\n        \n    def forward(self, inputs):\n        target_hist, ct_hist, content_hist, part_hist, time_hist, tcid_hist, answer_hist, ethe_hist, content, part, numeric = inputs\n        \n        tcid_hist = torch.log(1 + torch.clamp(tcid_hist, 0, None))\n        \n        content_difficulty_hist = self.content_difficulty_emb(content_hist)\n        content_difficulty_hist = torch.log(torch.clamp(content_difficulty_hist, 3, None))\n        part_hist = self.part_emb(part_hist)\n        \n        content_difficulty = self.content_difficulty_emb(content).squeeze(1)\n        content_difficulty = torch.log(torch.clamp(content_difficulty, 3, None))\n        part = self.part_emb(part)\n        \n        \n        content_similarity_hist = torch.cat([self.content_emb(content_hist), part_hist], -1)\n        content_similarity = torch.cat([self.content_emb(content), part], -1)\n        \n        \n        sim_features = []\n        for i in range(self.num_heads):\n            a = l2norm(self.sim_convs[i](content_similarity_hist))\n            b = l2norm(self.sim_convs[i](content_similarity)).repeat(1, content_similarity_hist.shape[1], 1)\n            sim_features.append((a*b).sum(axis=2))\n        \n        ca_hist = l2norm(self.content_answer_emb(content_hist))\n        a_hist = l2norm(self.answer_emb(answer_hist))\n        ca_hist = (ca_hist*a_hist).sum(axis=2)\n        \n        x = torch.cat([content_difficulty_hist, ethe_hist, content_similarity_hist] + \n                      [x.unsqueeze(-1) for x in [target_hist, ct_hist, time_hist, ca_hist, tcid_hist] + sim_features], \n                      axis=2)\n\n        x = self.hidden(x)*self.gate(x)\n        x_inv, _ = self.inv_gru(torch.flip(x, (1,)))\n        x = torch.cat([x, torch.flip(x_inv, (1,))], -1)\n        x, _ = self.gru(x)\n        x_inv, _ = self.inv_gru2(torch.flip(x, (1,)))\n        x = torch.cat([x, torch.flip(x_inv, (1,))], -1)\n        x, _ = self.gru2(x)\n        x = torch.cat([x[:, -1], content_difficulty, content_similarity.squeeze(1), numeric], axis=1)\n        x = self.final(x)\n        return x\n    \n    \nclass RidEnsembleModel(nn.Module):\n    def __init__(self, versions):\n        super(RidEnsembleModel, self).__init__()\n        self.models = []\n        for v in versions:\n            model = RidModel()\n            state_dict = torch.load(f\"/kaggle/input/riiid-models/{v}.pth\")\n            new_state_dict = OrderedDict()\n            for k, v in state_dict.items():\n                name = k[7:] # remove `module.`\n                new_state_dict[name] = v\n            model.load_state_dict(new_state_dict)\n            self.models.append(model)\n        self.models = nn.ModuleList(self.models)\n        \n    def update_difficulty_emb(self, emb):\n        emb = torch.Tensor(emb).cuda()\n        for i in range(len(self.models)):\n            self.models[i].content_difficulty_emb.weight += emb\n        \n    def forward(self, x):\n        return sum([model(x).sigmoid() for model in self.models])/len(self.models)\n    \nmodel = RidEnsembleModel([\"v56_0\", \"v56_1\", \"v56_2\", \"v56_3\", \"v56_4\"])\nmodel = model.cuda()\n\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:44.434596Z","iopub.status.busy":"2021-01-05T12:31:44.419032Z","iopub.status.idle":"2021-01-05T12:31:44.440498Z","shell.execute_reply":"2021-01-05T12:31:44.440962Z"},"papermill":{"duration":0.043309,"end_time":"2021-01-05T12:31:44.441079","exception":false,"start_time":"2021-01-05T12:31:44.39777","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\n\n\nclass RidInferenceDataset(Dataset):\n\n    def __init__(self, test_df, max_seq_len=256):\n        self.test_df = test_df\n        #self.test_df[\"prior_question_elapsed_time\"].fillna(-1000, inplace=True)\n        #self.test_df[\"prior_question_had_explanation\"].fillna(False, inplace=True)\n        self.max_seq_len = max_seq_len\n        \n    def _pad(self, array, pad_val):\n        if len(array) >= self.max_seq_len:\n            return array[-self.max_seq_len:]\n        shape = list(array.shape)\n        shape[0] = self.max_seq_len\n        x = np.ones(shape, dtype=np.float32)*pad_val\n        if len(array) > 0:\n            x[-len(array):] = array\n        return x\n\n    def __getitem__(self, idx):\n        row = self.test_df.iloc[idx].copy()\n        \n        user_df = dict(user_cache[row[\"user_id\"]]) # copy\n            \n        row[\"part\"] = qpart_dict[row[\"content_id\"]]\n\n        #df.sort_values(\"timestamp\", inplace=True)\n        \n        if user_df[\"ethe_hist\"] is not None:\n            ethe_hist = np.log1p(user_df[\"ethe_hist\"])\n        else:\n            ethe_hist = np.log1p(np.zeros((1, 3), dtype=np.float32))\n        \n        times = np.log1p((row[\"timestamp\"] - user_df[\"timestamp\"])*1e-6)\n        \n        tcid_hist = user_df[\"task_container_id\"] - row[\"task_container_id\"]\n        \n        content_type_hist = user_df[\"content_type_id\"]\n        user_answer = user_df[\"user_answer\"] + 1\n        correct_answer = CA[row[\"content_id\"]] + 1\n        \n        numeric = np.zeros(3)\n        valid_answers = user_answer[content_type_hist == 0][::-1]\n        last_time_same_answer = np.where(valid_answers == correct_answer)[0]\n        if len(last_time_same_answer) > 0:\n            last_time_same_answer = last_time_same_answer[0]\n        else:\n            last_time_same_answer = len(valid_answers)\n        numeric[0] = last_time_same_answer\n        \n        if len(valid_answers) > 0:\n            last_answer = valid_answers[0]\n            seq_len = np.where(valid_answers != last_answer)[0]\n            if len(seq_len) > 0:\n                seq_len = seq_len[0]\n            else:\n                seq_len = 1\n            if last_answer == correct_answer:\n                numeric[1] = seq_len\n            else:\n                numeric[2] = seq_len\n                \n        numeric = np.log1p(numeric)\n        \n        outputs = {\"target_hist\": (1 - content_type_hist)*(2*user_df[\"answered_correctly\"] - 1),\n                   \"content_type_hist\": content_type_hist,\n                   \"content_hist\": user_df[\"content_id\"],\n                   \"part_hist\": user_df[\"part\"],\n                   \"time_hist\": times,\n                   \"tcid_hist\": tcid_hist,\n                   \"answer_hist\": user_answer,\n                   \"ethe_hist\": ethe_hist,\n                   \"content\": [row[\"content_id\"]],\n                   \"part\": [row[\"part\"]],\n                   \"numeric\": numeric\n                  }\n        \n        for key in outputs.keys():\n            if \"hist\" in key:\n                outputs[key] = self._pad(outputs[key], 0)\n            if key in {\"content_hist\", \"content\", \"part_hist\", \"part\", \"answer_hist\"}:\n                outputs[key] = torch.LongTensor(outputs[key])\n            else:\n                outputs[key] = torch.FloatTensor(outputs[key])\n        \n        return tuple(o for o in outputs.values())\n\n\n    def __len__(self):\n        return len(self.test_df)\n    \n#example_dataset = RidInferenceDataset(test_df)\n#example_dataset[0]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:44.50098Z","iopub.status.busy":"2021-01-05T12:31:44.495474Z","iopub.status.idle":"2021-01-05T12:31:44.507838Z","shell.execute_reply":"2021-01-05T12:31:44.507346Z"},"papermill":{"duration":0.054857,"end_time":"2021-01-05T12:31:44.507934","exception":false,"start_time":"2021-01-05T12:31:44.453077","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"columns = {\n        'timestamp': 'int64',\n        'user_id': 'int32',\n        'content_id': 'int16',\n        'content_type_id': 'int8',\n        'task_container_id': 'int16',\n        'user_answer': 'int8',\n        'answered_correctly':'int8',\n        'prior_question_elapsed_time': 'float32',\n        'prior_question_had_explanation': 'boolean'\n    }\n\ndef concat(arr1, arr2):\n    if arr1 is None:\n        return arr2\n    if len(arr1) == 0:\n        return arr2\n    return np.concatenate([arr1, arr2])\n\n\ndef read_ethe(user_dict):\n    ethe_hist = np.zeros((len(user_dict[\"content_id\"]), 3), dtype=np.float32)*np.nan\n    pqet = user_dict['prior_question_elapsed_time'][-1]\n    pqhe = user_dict['prior_question_had_explanation'][-1]\n    pqet_hist = user_dict[\"prior_question_elapsed_time\"]\n    pqhe_hist = user_dict['prior_question_had_explanation']\n    container = user_dict[\"task_container_id\"][-1]\n    \n    pqts_hist = user_dict['timestamp']*1e-6\n    pqts = pqts_hist[-1]\n\n    for i in range(1, ethe_hist.shape[0]):\n        ix = -i - 1\n        \n        if user_dict[\"task_container_id\"][ix] == container:\n            ethe_hist[ix] = ethe_hist[ix + 1]\n        else:\n            container = user_dict[\"task_container_id\"][ix]\n            if pd.isna(pqet):\n                pqet = 0.0\n            if pd.isna(pqhe):\n                pqhe = False\n            ethe_hist[ix, 0] = pqet*1e-6\n            ethe_hist[ix, 1] = 1.0*pqhe\n            ethe_hist[ix, 2] = pqts - pqts_hist[ix]\n            pqet = pqet_hist[ix]\n            pqhe = pqhe_hist[ix]\n            pqts = pqts_hist[ix]\n\n    return ethe_hist       \n    \n\ndef update_ethe(df, users):\n    for user in users:\n        user_ethe = user_cache[user][\"ethe_hist\"]\n        new_user_data = df[df[\"user_id\"] == user].copy()\n        \n        if (new_user_data.shape[0] == 0) or (user_ethe is None):\n            continue\n        \n        pqet = (new_user_data[\"prior_question_elapsed_time\"].fillna(0.0)*1e-6).values[0]\n        pqhe = (new_user_data[\"prior_question_had_explanation\"].fillna(False)*1.0).values[0]\n        \n        ts = user_cache[user][\"timestamp\"]\n        pqts = new_user_data[\"timestamp\"].values[0]\n        \n        for i in range(user_ethe.shape[0]):\n            ix = -1 - i\n            if np.isnan(user_ethe[ix, 2]):\n                user_ethe[ix, 0] = pqet\n                user_ethe[ix, 1] = pqhe\n                user_ethe[ix, 2] = 1e-6*(pqts - ts[ix])\n            else:\n                break\n        \n        user_cache[user][\"ethe_hist\"] = user_ethe\n\n\ndef update_user_cache_q(df, users):\n    new_target = np.zeros((lid_max, 2))\n    for user in users:\n        new_user_data = df[df[\"user_id\"] == user].copy()\n        \n        if new_user_data.shape[0] > 0:\n            user_data = user_cache[user]\n        \n            content_id = new_user_data[\"content_id\"].values\n            target = new_user_data[\"answered_correctly\"].values\n\n            new_user_data[\"part\"] = np.array([qpart_dict[x] for x in content_id])\n            new_user_data[\"content_id\"] = content_id\n\n            for key in list(columns.keys()) + [\"part\"]:\n                user_data[key] = concat(user_data[key], new_user_data[key].values)\n\n            user_data[\"ethe_hist\"] = concat(user_data[\"ethe_hist\"], np.zeros((len(content_id), 3), dtype=np.float32)*np.nan)\n\n            user_cache[user] = user_data\n\n            for q in range(new_user_data.shape[0]):\n                new_target[content_id[q], target[q]] += 1\n\n    with torch.no_grad():\n        model.update_difficulty_emb(new_target)\n\n        \ndef hash_folder(x, k=16):\n    name = \"\"\n    for i in range(3):\n        name = name + f\"{x%k}/\"\n        x = x // k\n    return name\n\ndef read_user_cache(users):\n    for user in users:\n        if user not in user_cache:\n            folder = f\"/kaggle/input/riiid-partitioned/npdata/{hash_folder(user)}\"\n            filename = f\"{folder}{user}.npz\"\n            \n            user_dict = dict()\n            \n            try:\n                user_data = np.load(filename, allow_pickle=True)\n                for key in columns.keys():\n                    user_dict[key] = user_data[key]\n                #print(\"Found:\", filename)\n\n            except:\n                #print(f\"Not found {filename}, new user.\")\n                for key in columns.keys():\n                    user_dict[key] = np.array([])\n                \n            questions = np.where(user_dict[\"content_type_id\"] == 0)[0]\n            \n            for key in columns.keys():\n                user_dict[key] = user_dict[key][questions]\n\n            user_dict[\"part\"] = np.array([qpart_dict[x] for x in user_dict[\"content_id\"]])\n            \n            if len(user_dict[\"part\"]) > 0:\n                user_dict[\"ethe_hist\"] = read_ethe(user_dict)\n            else:\n                user_dict[\"ethe_hist\"] = None\n            user_cache[user] = user_dict","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:44.538956Z","iopub.status.busy":"2021-01-05T12:31:44.538129Z","iopub.status.idle":"2021-01-05T12:31:44.540584Z","shell.execute_reply":"2021-01-05T12:31:44.541056Z"},"papermill":{"duration":0.021263,"end_time":"2021-01-05T12:31:44.541169","exception":false,"start_time":"2021-01-05T12:31:44.519906","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data)\n\ndef predict(eval_loader):\n    \n    preds = []\n\n    for idx, data in enumerate(eval_loader):\n        inputs = read_data(data)\n\n        pred = model(inputs)\n\n        preds.append(pred.detach().cpu().numpy().ravel())\n    \n    return np.concatenate(preds)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-05T12:31:44.57658Z","iopub.status.busy":"2021-01-05T12:31:44.575875Z","iopub.status.idle":"2021-01-05T12:31:47.213181Z","shell.execute_reply":"2021-01-05T12:31:47.212296Z"},"papermill":{"duration":2.66005,"end_time":"2021-01-05T12:31:47.213308","exception":false,"start_time":"2021-01-05T12:31:44.553258","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TARGET = \"answered_correctly\"\nBS = 64\nNW = 1\n\n\nprevious_test_df = None\n\niter_test = tqdm(env.iter_test())\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        previous_test_df[\"user_answer\"] = eval(test_df[\"prior_group_responses\"].iloc[0])\n        update_user_cache_q(previous_test_df[previous_test_df['content_type_id'] == 0].reset_index(drop=True), relevant_users)\n    \n    relevant_users = np.unique(test_df[\"user_id\"].values)\n    read_user_cache(relevant_users)\n\n    eval_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    update_ethe(eval_df, relevant_users)\n    \n    eval_dataset = RidInferenceDataset(eval_df)\n    eval_loader = DataLoader(eval_dataset, batch_size=BS, shuffle=False, num_workers=NW, pin_memory=False, drop_last=False)\n\n    eval_df[TARGET] = predict(eval_loader)\n    \n    env.predict(eval_df[['row_id', TARGET]])\n    \n    previous_test_df = test_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.014592,"end_time":"2021-01-05T12:31:47.242527","exception":false,"start_time":"2021-01-05T12:31:47.227935","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}